@inproceedings{braun-etal-2026-two,
title = "Two Lawyers, Three Opinions: Can Output-Based {LLM} Uncertainty Find the Cases Where Experts Disagree?",
author = "Braun, Laura and
A{\ss}enmacher, Matthias and
Hohenadler, Martin and
Kauermann, Goeran",
editor = {Aziz, Wilker and
Berant, Jonathan and
Eikema, Bryan and
Ilia, Evgenia and
de Marneffe, Marie-Catherine and
Plank, Barbara and
Shelmanov, Artem and
Swayamdipta, Swabha and
Tiedemann, J{\"o}rg and
Vazhentsev, Artem and
V{\'a}zquez, Ra{\'u}l and
Zerva, Chrysoula},
booktitle = "The Proceedings of the Third Workshop on Uncertainty-Aware {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.uncertainlp-main.19/",
pages = "301--311",
ISBN = "979-8-89176-486-6",
abstract = "Legal natural language processing (NLP) tasks often lack a single objective ground truth, as reflected in label variation across experts' annotations. We test whether two uncertainty signals based on outputs of large language models (LLMs), sampling entropy and verbalized probability, can identify posts where legal experts disagree. On 1,564 German social media posts annotated by three legal experts for punishability under the German Criminal Code, we evaluate four open-weight LLMs, with reasoning either enabled or disabled, and observe a drop in F1 score on contested posts. Sampling distributions collapse to near-deterministic judgments in most conditions, while verbalized probabilities track expert disagreement in aggregate and support deferral of model errors. Neither signal, however, reliably identifies individual contested posts: both detect the model{'}s own errors (AUROC up to 0.83) better than expert disagreement (up to 0.76), and which signal is more informative depends strongly on model and activation of reasoning mode."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="braun-etal-2026-two">
<titleInfo>
<title>Two Lawyers, Three Opinions: Can Output-Based LLM Uncertainty Find the Cases Where Experts Disagree?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Laura</namePart>
<namePart type="family">Braun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Matthias</namePart>
<namePart type="family">Aßenmacher</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Hohenadler</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Goeran</namePart>
<namePart type="family">Kauermann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>The Proceedings of the Third Workshop on Uncertainty-Aware NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wilker</namePart>
<namePart type="family">Aziz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jonathan</namePart>
<namePart type="family">Berant</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bryan</namePart>
<namePart type="family">Eikema</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Evgenia</namePart>
<namePart type="family">Ilia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marie-Catherine</namePart>
<namePart type="family">de Marneffe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Artem</namePart>
<namePart type="family">Shelmanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Swabha</namePart>
<namePart type="family">Swayamdipta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jörg</namePart>
<namePart type="family">Tiedemann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Artem</namePart>
<namePart type="family">Vazhentsev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raúl</namePart>
<namePart type="family">Vázquez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chrysoula</namePart>
<namePart type="family">Zerva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-486-6</identifier>
</relatedItem>
<abstract>Legal natural language processing (NLP) tasks often lack a single objective ground truth, as reflected in label variation across experts’ annotations. We test whether two uncertainty signals based on outputs of large language models (LLMs), sampling entropy and verbalized probability, can identify posts where legal experts disagree. On 1,564 German social media posts annotated by three legal experts for punishability under the German Criminal Code, we evaluate four open-weight LLMs, with reasoning either enabled or disabled, and observe a drop in F1 score on contested posts. Sampling distributions collapse to near-deterministic judgments in most conditions, while verbalized probabilities track expert disagreement in aggregate and support deferral of model errors. Neither signal, however, reliably identifies individual contested posts: both detect the model’s own errors (AUROC up to 0.83) better than expert disagreement (up to 0.76), and which signal is more informative depends strongly on model and activation of reasoning mode.</abstract>
<identifier type="citekey">braun-etal-2026-two</identifier>
<location>
<url>https://aclanthology.org/2026.uncertainlp-main.19/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>301</start>
<end>311</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Two Lawyers, Three Opinions: Can Output-Based LLM Uncertainty Find the Cases Where Experts Disagree?
%A Braun, Laura
%A Aßenmacher, Matthias
%A Hohenadler, Martin
%A Kauermann, Goeran
%Y Aziz, Wilker
%Y Berant, Jonathan
%Y Eikema, Bryan
%Y Ilia, Evgenia
%Y de Marneffe, Marie-Catherine
%Y Plank, Barbara
%Y Shelmanov, Artem
%Y Swayamdipta, Swabha
%Y Tiedemann, Jörg
%Y Vazhentsev, Artem
%Y Vázquez, Raúl
%Y Zerva, Chrysoula
%S The Proceedings of the Third Workshop on Uncertainty-Aware NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-486-6
%F braun-etal-2026-two
%X Legal natural language processing (NLP) tasks often lack a single objective ground truth, as reflected in label variation across experts’ annotations. We test whether two uncertainty signals based on outputs of large language models (LLMs), sampling entropy and verbalized probability, can identify posts where legal experts disagree. On 1,564 German social media posts annotated by three legal experts for punishability under the German Criminal Code, we evaluate four open-weight LLMs, with reasoning either enabled or disabled, and observe a drop in F1 score on contested posts. Sampling distributions collapse to near-deterministic judgments in most conditions, while verbalized probabilities track expert disagreement in aggregate and support deferral of model errors. Neither signal, however, reliably identifies individual contested posts: both detect the model’s own errors (AUROC up to 0.83) better than expert disagreement (up to 0.76), and which signal is more informative depends strongly on model and activation of reasoning mode.
%U https://aclanthology.org/2026.uncertainlp-main.19/
%P 301-311
Markdown (Informal)
[Two Lawyers, Three Opinions: Can Output-Based LLM Uncertainty Find the Cases Where Experts Disagree?](https://aclanthology.org/2026.uncertainlp-main.19/) (Braun et al., UncertaiNLP 2026)
ACL