@inproceedings{pejcinovic-etal-2026-isnt,
title = "``But Isn{'}t It All Just Semantics?'' The Importance of Evaluating Uncertainty, Reliability, and Random Effects",
author = "Pejcinovic, Katarina and
Casilio, Marianne and
Gale, Robert C and
Fergadiotis, Gerasimos and
Bedrick, Steven",
editor = {Aziz, Wilker and
Berant, Jonathan and
Eikema, Bryan and
Ilia, Evgenia and
de Marneffe, Marie-Catherine and
Plank, Barbara and
Shelmanov, Artem and
Swayamdipta, Swabha and
Tiedemann, J{\"o}rg and
Vazhentsev, Artem and
V{\'a}zquez, Ra{\'u}l and
Zerva, Chrysoula},
booktitle = "The Proceedings of the Third Workshop on Uncertainty-Aware {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.uncertainlp-main.25/",
pages = "408--419",
ISBN = "979-8-89176-486-6",
abstract = "Picture naming tests, which involve showing images and asking patients to name them, are widely used clinical and research tools to evaluate a patient{'}s ability to retrieve and produce words. One essential aspect of such tests is categorizing the type of semantic similarity between the image and the patient{'}s response, as classifying errors is important for treatment planning and prognosis. These tests are tedious to score and prone to inter-rater variability, both of which could be improved with automation via LLMs. This task highlights the clinical potential of LLMs, but also the need for robust statistical testing of LLM performance. In this work, we show that traditional evaluation methods can be misleading for complex experimental designs. Using generalized linear mixed effects models and conformal prediction for uncertainty quantification, we identify sources of variation in model performance and estimate the uncertainty of model predictions. We show that seemingly large differences in model performance from traditional evaluations are misleading, driven instead by item-level and category difficulty. Additionally, uncertainty in predictions varies substantially between semantic categories. These results suggest that for NLP evaluations with nested, clustered, or uneven category distributions, uncertainty quantification can be necessary to avoid misleading conclusions for high-stakes clinical tasks."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="pejcinovic-etal-2026-isnt">
<titleInfo>
<title>“But Isn’t It All Just Semantics?” The Importance of Evaluating Uncertainty, Reliability, and Random Effects</title>
</titleInfo>
<name type="personal">
<namePart type="given">Katarina</namePart>
<namePart type="family">Pejcinovic</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marianne</namePart>
<namePart type="family">Casilio</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Robert</namePart>
<namePart type="given">C</namePart>
<namePart type="family">Gale</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gerasimos</namePart>
<namePart type="family">Fergadiotis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Steven</namePart>
<namePart type="family">Bedrick</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>The Proceedings of the Third Workshop on Uncertainty-Aware NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wilker</namePart>
<namePart type="family">Aziz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jonathan</namePart>
<namePart type="family">Berant</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bryan</namePart>
<namePart type="family">Eikema</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Evgenia</namePart>
<namePart type="family">Ilia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marie-Catherine</namePart>
<namePart type="family">de Marneffe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Artem</namePart>
<namePart type="family">Shelmanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Swabha</namePart>
<namePart type="family">Swayamdipta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jörg</namePart>
<namePart type="family">Tiedemann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Artem</namePart>
<namePart type="family">Vazhentsev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raúl</namePart>
<namePart type="family">Vázquez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chrysoula</namePart>
<namePart type="family">Zerva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-486-6</identifier>
</relatedItem>
<abstract>Picture naming tests, which involve showing images and asking patients to name them, are widely used clinical and research tools to evaluate a patient’s ability to retrieve and produce words. One essential aspect of such tests is categorizing the type of semantic similarity between the image and the patient’s response, as classifying errors is important for treatment planning and prognosis. These tests are tedious to score and prone to inter-rater variability, both of which could be improved with automation via LLMs. This task highlights the clinical potential of LLMs, but also the need for robust statistical testing of LLM performance. In this work, we show that traditional evaluation methods can be misleading for complex experimental designs. Using generalized linear mixed effects models and conformal prediction for uncertainty quantification, we identify sources of variation in model performance and estimate the uncertainty of model predictions. We show that seemingly large differences in model performance from traditional evaluations are misleading, driven instead by item-level and category difficulty. Additionally, uncertainty in predictions varies substantially between semantic categories. These results suggest that for NLP evaluations with nested, clustered, or uneven category distributions, uncertainty quantification can be necessary to avoid misleading conclusions for high-stakes clinical tasks.</abstract>
<identifier type="citekey">pejcinovic-etal-2026-isnt</identifier>
<location>
<url>https://aclanthology.org/2026.uncertainlp-main.25/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>408</start>
<end>419</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T “But Isn’t It All Just Semantics?” The Importance of Evaluating Uncertainty, Reliability, and Random Effects
%A Pejcinovic, Katarina
%A Casilio, Marianne
%A Gale, Robert C.
%A Fergadiotis, Gerasimos
%A Bedrick, Steven
%Y Aziz, Wilker
%Y Berant, Jonathan
%Y Eikema, Bryan
%Y Ilia, Evgenia
%Y de Marneffe, Marie-Catherine
%Y Plank, Barbara
%Y Shelmanov, Artem
%Y Swayamdipta, Swabha
%Y Tiedemann, Jörg
%Y Vazhentsev, Artem
%Y Vázquez, Raúl
%Y Zerva, Chrysoula
%S The Proceedings of the Third Workshop on Uncertainty-Aware NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-486-6
%F pejcinovic-etal-2026-isnt
%X Picture naming tests, which involve showing images and asking patients to name them, are widely used clinical and research tools to evaluate a patient’s ability to retrieve and produce words. One essential aspect of such tests is categorizing the type of semantic similarity between the image and the patient’s response, as classifying errors is important for treatment planning and prognosis. These tests are tedious to score and prone to inter-rater variability, both of which could be improved with automation via LLMs. This task highlights the clinical potential of LLMs, but also the need for robust statistical testing of LLM performance. In this work, we show that traditional evaluation methods can be misleading for complex experimental designs. Using generalized linear mixed effects models and conformal prediction for uncertainty quantification, we identify sources of variation in model performance and estimate the uncertainty of model predictions. We show that seemingly large differences in model performance from traditional evaluations are misleading, driven instead by item-level and category difficulty. Additionally, uncertainty in predictions varies substantially between semantic categories. These results suggest that for NLP evaluations with nested, clustered, or uneven category distributions, uncertainty quantification can be necessary to avoid misleading conclusions for high-stakes clinical tasks.
%U https://aclanthology.org/2026.uncertainlp-main.25/
%P 408-419
Markdown (Informal)
["But Isn’t It All Just Semantics?" The Importance of Evaluating Uncertainty, Reliability, and Random Effects](https://aclanthology.org/2026.uncertainlp-main.25/) (Pejcinovic et al., UncertaiNLP 2026)
ACL