@inproceedings{kim-etal-2026-uncertainty,
title = "Uncertainty Meets Conformance: A Process Mining-based Multi-Objective Evaluation for Clinical Reasoning using {LLM}s",
author = "Kim, Seon and
Lee, Jeong-woo and
Hoon Kong, Tae and
Kim, Jongchan",
editor = {Aziz, Wilker and
Berant, Jonathan and
Eikema, Bryan and
Ilia, Evgenia and
de Marneffe, Marie-Catherine and
Plank, Barbara and
Shelmanov, Artem and
Swayamdipta, Swabha and
Tiedemann, J{\"o}rg and
Vazhentsev, Artem and
V{\'a}zquez, Ra{\'u}l and
Zerva, Chrysoula},
booktitle = "The Proceedings of the Third Workshop on Uncertainty-Aware {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.uncertainlp-main.23/",
pages = "381--389",
ISBN = "979-8-89176-486-6",
abstract = "We present a framework for evaluating LLM-based clinical reasoning using conformance metrics from process mining. Four local LLMs were tested on EMR data from 16 vertigo patients across four prompting methods, with five repetitions per combination, yielding six uncertainty metrics. Reasoning traces were converted into event logs and analyzed with Heuristic Miner to derive external conformance to other patients' reasoning patterns and internal conformance across repeated runs. Treating these as joint objectives, Pareto, Hypervolume, and TOPSIS analyses identified distinct evaluation scenarios, showing that conformance metrics can complement outcome-based uncertainty by revealing how LLMs reach their conclusions."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kim-etal-2026-uncertainty">
<titleInfo>
<title>Uncertainty Meets Conformance: A Process Mining-based Multi-Objective Evaluation for Clinical Reasoning using LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Seon</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jeong-woo</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tae</namePart>
<namePart type="family">Hoon Kong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jongchan</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>The Proceedings of the Third Workshop on Uncertainty-Aware NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wilker</namePart>
<namePart type="family">Aziz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jonathan</namePart>
<namePart type="family">Berant</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bryan</namePart>
<namePart type="family">Eikema</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Evgenia</namePart>
<namePart type="family">Ilia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marie-Catherine</namePart>
<namePart type="family">de Marneffe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Artem</namePart>
<namePart type="family">Shelmanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Swabha</namePart>
<namePart type="family">Swayamdipta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jörg</namePart>
<namePart type="family">Tiedemann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Artem</namePart>
<namePart type="family">Vazhentsev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raúl</namePart>
<namePart type="family">Vázquez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chrysoula</namePart>
<namePart type="family">Zerva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-486-6</identifier>
</relatedItem>
<abstract>We present a framework for evaluating LLM-based clinical reasoning using conformance metrics from process mining. Four local LLMs were tested on EMR data from 16 vertigo patients across four prompting methods, with five repetitions per combination, yielding six uncertainty metrics. Reasoning traces were converted into event logs and analyzed with Heuristic Miner to derive external conformance to other patients’ reasoning patterns and internal conformance across repeated runs. Treating these as joint objectives, Pareto, Hypervolume, and TOPSIS analyses identified distinct evaluation scenarios, showing that conformance metrics can complement outcome-based uncertainty by revealing how LLMs reach their conclusions.</abstract>
<identifier type="citekey">kim-etal-2026-uncertainty</identifier>
<location>
<url>https://aclanthology.org/2026.uncertainlp-main.23/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>381</start>
<end>389</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Uncertainty Meets Conformance: A Process Mining-based Multi-Objective Evaluation for Clinical Reasoning using LLMs
%A Kim, Seon
%A Lee, Jeong-woo
%A Hoon Kong, Tae
%A Kim, Jongchan
%Y Aziz, Wilker
%Y Berant, Jonathan
%Y Eikema, Bryan
%Y Ilia, Evgenia
%Y de Marneffe, Marie-Catherine
%Y Plank, Barbara
%Y Shelmanov, Artem
%Y Swayamdipta, Swabha
%Y Tiedemann, Jörg
%Y Vazhentsev, Artem
%Y Vázquez, Raúl
%Y Zerva, Chrysoula
%S The Proceedings of the Third Workshop on Uncertainty-Aware NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-486-6
%F kim-etal-2026-uncertainty
%X We present a framework for evaluating LLM-based clinical reasoning using conformance metrics from process mining. Four local LLMs were tested on EMR data from 16 vertigo patients across four prompting methods, with five repetitions per combination, yielding six uncertainty metrics. Reasoning traces were converted into event logs and analyzed with Heuristic Miner to derive external conformance to other patients’ reasoning patterns and internal conformance across repeated runs. Treating these as joint objectives, Pareto, Hypervolume, and TOPSIS analyses identified distinct evaluation scenarios, showing that conformance metrics can complement outcome-based uncertainty by revealing how LLMs reach their conclusions.
%U https://aclanthology.org/2026.uncertainlp-main.23/
%P 381-389
Markdown (Informal)
[Uncertainty Meets Conformance: A Process Mining-based Multi-Objective Evaluation for Clinical Reasoning using LLMs](https://aclanthology.org/2026.uncertainlp-main.23/) (Kim et al., UncertaiNLP 2026)
ACL