@inproceedings{zangrillo-etal-2026-paint,
title = "Paint It, Black: Evaluating {LLM}s for De-Identification of Multimodal Clinical Documents",
author = "Zangrillo, Francesco and
Cocchieri, Alessio and
Frisoni, Giacomo and
Freddi, Davide and
Ragazzi, Luca and
Aguzzi, Alberto and
Raimondo, Diego and
Seracchioli, Renato and
Moro, Gianluca",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.561/",
pages = "12865--12902",
ISBN = "979-8-89176-459-0",
abstract = "Secondary use of clinical data requires removing Protected Health Information (PHI) under regulations such as HIPAA. While de-identification is well studied for text, clinical records are often structured PDFs, where PHI appears across complex layouts, tables, and headers. We introduce Paint-It-Black (PIB), the first benchmark for end-to-end, pixel-level PHI de-identification in clinical PDFs. PIB contains physician-validated synthetic documents with diverse layouts, document types, and controlled visual degradations, annotated with HIPAA-aligned labels and bounding boxes. We evaluate locally deployable Multimodal Large Language Models (MLLMs) and find that even the strongest remain below the reliability required for clinical use. A conventional OCR+NER baseline performs substantially worse than the best MLLMs, highlighting the inherently multimodal nature of clinical PHI de-identification."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zangrillo-etal-2026-paint">
<titleInfo>
<title>Paint It, Black: Evaluating LLMs for De-Identification of Multimodal Clinical Documents</title>
</titleInfo>
<name type="personal">
<namePart type="given">Francesco</namePart>
<namePart type="family">Zangrillo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alessio</namePart>
<namePart type="family">Cocchieri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Giacomo</namePart>
<namePart type="family">Frisoni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Davide</namePart>
<namePart type="family">Freddi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Luca</namePart>
<namePart type="family">Ragazzi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alberto</namePart>
<namePart type="family">Aguzzi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Diego</namePart>
<namePart type="family">Raimondo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Renato</namePart>
<namePart type="family">Seracchioli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gianluca</namePart>
<namePart type="family">Moro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Secondary use of clinical data requires removing Protected Health Information (PHI) under regulations such as HIPAA. While de-identification is well studied for text, clinical records are often structured PDFs, where PHI appears across complex layouts, tables, and headers. We introduce Paint-It-Black (PIB), the first benchmark for end-to-end, pixel-level PHI de-identification in clinical PDFs. PIB contains physician-validated synthetic documents with diverse layouts, document types, and controlled visual degradations, annotated with HIPAA-aligned labels and bounding boxes. We evaluate locally deployable Multimodal Large Language Models (MLLMs) and find that even the strongest remain below the reliability required for clinical use. A conventional OCR+NER baseline performs substantially worse than the best MLLMs, highlighting the inherently multimodal nature of clinical PHI de-identification.</abstract>
<identifier type="citekey">zangrillo-etal-2026-paint</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.561/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>12865</start>
<end>12902</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Paint It, Black: Evaluating LLMs for De-Identification of Multimodal Clinical Documents
%A Zangrillo, Francesco
%A Cocchieri, Alessio
%A Frisoni, Giacomo
%A Freddi, Davide
%A Ragazzi, Luca
%A Aguzzi, Alberto
%A Raimondo, Diego
%A Seracchioli, Renato
%A Moro, Gianluca
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F zangrillo-etal-2026-paint
%X Secondary use of clinical data requires removing Protected Health Information (PHI) under regulations such as HIPAA. While de-identification is well studied for text, clinical records are often structured PDFs, where PHI appears across complex layouts, tables, and headers. We introduce Paint-It-Black (PIB), the first benchmark for end-to-end, pixel-level PHI de-identification in clinical PDFs. PIB contains physician-validated synthetic documents with diverse layouts, document types, and controlled visual degradations, annotated with HIPAA-aligned labels and bounding boxes. We evaluate locally deployable Multimodal Large Language Models (MLLMs) and find that even the strongest remain below the reliability required for clinical use. A conventional OCR+NER baseline performs substantially worse than the best MLLMs, highlighting the inherently multimodal nature of clinical PHI de-identification.
%U https://aclanthology.org/2026.emnlp-main.561/
%P 12865-12902
Markdown (Informal)
[Paint It, Black: Evaluating LLMs for De-Identification of Multimodal Clinical Documents](https://aclanthology.org/2026.emnlp-main.561/) (Zangrillo et al., EMNLP 2026)
ACL
- Francesco Zangrillo, Alessio Cocchieri, Giacomo Frisoni, Davide Freddi, Luca Ragazzi, Alberto Aguzzi, Diego Raimondo, Renato Seracchioli, and Gianluca Moro. 2026. Paint It, Black: Evaluating LLMs for De-Identification of Multimodal Clinical Documents. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 12865–12902, Budapest, Hungary. Association for Computational Linguistics.