@inproceedings{rizzi-etal-2026-evaluating,
title = "Evaluating with Disagreement: Metrics for Soft and Perspectivist Evaluation of Categorical and Ordinal Judgments",
author = "Rizzi, Giulia and
Casola, Silvia and
Fersini, Elisabetta and
Leonardelli, Elisa and
Plank, Barbara and
Poesio, Massimo",
editor = "Xu, Shanshan and
T.Y.S.S., Santosh and
Kurniawan, Kemal and
Peng, Siyao and
Wallbridge, Sarenne and
Koto, Fajri and
Dziri, Nouha and
Plank, Barbara",
booktitle = "Proceedings of the First Workshop on Pluralistic {AI} {NLP}: Diversity-aware, Sociotechnical, Responsible Alignment ({PANDORA} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.pandora-1.14/",
pages = "250--265",
ISBN = "979-8-89176-484-2",
abstract = "Disagreement-aware NLP challenges the assumption that each instance has a single gold label and raises new questions about evaluation. We examine the evaluation framework introduced in the third edition of the Learning With Disagreement (LeWiDi) shared task, which combines soft evaluation of label distributions with perspectivist evaluation of annotator-specific judgments. We describe the metrics adopted and motivate their adaptation to categorical, ordinal, single-label, and multilabel data. We then analyse their desirable properties and limitations, extending previous work to ordinal and annotator-level evaluation. Finally, using the LeWiDi 2025 submissions, we empirically study the relationship between the soft and perspectivist settings for the submitted systems by distinguishing errors in predicting the overall pattern of labels from errors in assigning those predictions to specific annotators. Our results show that the selected metrics satisfy desiderata while involving important trade-offs. Soft and perspectivist evaluation are closely related but not interchangeable: for most submitted systems, most perspectivist error remains even after predictions are optimally reassigned across annotators, while the additional annotator-assignment error is smaller."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="rizzi-etal-2026-evaluating">
<titleInfo>
<title>Evaluating with Disagreement: Metrics for Soft and Perspectivist Evaluation of Categorical and Ordinal Judgments</title>
</titleInfo>
<name type="personal">
<namePart type="given">Giulia</namePart>
<namePart type="family">Rizzi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Silvia</namePart>
<namePart type="family">Casola</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elisabetta</namePart>
<namePart type="family">Fersini</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elisa</namePart>
<namePart type="family">Leonardelli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Massimo</namePart>
<namePart type="family">Poesio</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shanshan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Santosh</namePart>
<namePart type="family">T.Y.S.S.</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kemal</namePart>
<namePart type="family">Kurniawan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siyao</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sarenne</namePart>
<namePart type="family">Wallbridge</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fajri</namePart>
<namePart type="family">Koto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nouha</namePart>
<namePart type="family">Dziri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-484-2</identifier>
</relatedItem>
<abstract>Disagreement-aware NLP challenges the assumption that each instance has a single gold label and raises new questions about evaluation. We examine the evaluation framework introduced in the third edition of the Learning With Disagreement (LeWiDi) shared task, which combines soft evaluation of label distributions with perspectivist evaluation of annotator-specific judgments. We describe the metrics adopted and motivate their adaptation to categorical, ordinal, single-label, and multilabel data. We then analyse their desirable properties and limitations, extending previous work to ordinal and annotator-level evaluation. Finally, using the LeWiDi 2025 submissions, we empirically study the relationship between the soft and perspectivist settings for the submitted systems by distinguishing errors in predicting the overall pattern of labels from errors in assigning those predictions to specific annotators. Our results show that the selected metrics satisfy desiderata while involving important trade-offs. Soft and perspectivist evaluation are closely related but not interchangeable: for most submitted systems, most perspectivist error remains even after predictions are optimally reassigned across annotators, while the additional annotator-assignment error is smaller.</abstract>
<identifier type="citekey">rizzi-etal-2026-evaluating</identifier>
<location>
<url>https://aclanthology.org/2026.pandora-1.14/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>250</start>
<end>265</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating with Disagreement: Metrics for Soft and Perspectivist Evaluation of Categorical and Ordinal Judgments
%A Rizzi, Giulia
%A Casola, Silvia
%A Fersini, Elisabetta
%A Leonardelli, Elisa
%A Plank, Barbara
%A Poesio, Massimo
%Y Xu, Shanshan
%Y T.Y.S.S., Santosh
%Y Kurniawan, Kemal
%Y Peng, Siyao
%Y Wallbridge, Sarenne
%Y Koto, Fajri
%Y Dziri, Nouha
%Y Plank, Barbara
%S Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-484-2
%F rizzi-etal-2026-evaluating
%X Disagreement-aware NLP challenges the assumption that each instance has a single gold label and raises new questions about evaluation. We examine the evaluation framework introduced in the third edition of the Learning With Disagreement (LeWiDi) shared task, which combines soft evaluation of label distributions with perspectivist evaluation of annotator-specific judgments. We describe the metrics adopted and motivate their adaptation to categorical, ordinal, single-label, and multilabel data. We then analyse their desirable properties and limitations, extending previous work to ordinal and annotator-level evaluation. Finally, using the LeWiDi 2025 submissions, we empirically study the relationship between the soft and perspectivist settings for the submitted systems by distinguishing errors in predicting the overall pattern of labels from errors in assigning those predictions to specific annotators. Our results show that the selected metrics satisfy desiderata while involving important trade-offs. Soft and perspectivist evaluation are closely related but not interchangeable: for most submitted systems, most perspectivist error remains even after predictions are optimally reassigned across annotators, while the additional annotator-assignment error is smaller.
%U https://aclanthology.org/2026.pandora-1.14/
%P 250-265
Markdown (Informal)
[Evaluating with Disagreement: Metrics for Soft and Perspectivist Evaluation of Categorical and Ordinal Judgments](https://aclanthology.org/2026.pandora-1.14/) (Rizzi et al., PANDORA 2026)
ACL
- Giulia Rizzi, Silvia Casola, Elisabetta Fersini, Elisa Leonardelli, Barbara Plank, and Massimo Poesio. 2026. Evaluating with Disagreement: Metrics for Soft and Perspectivist Evaluation of Categorical and Ordinal Judgments. In Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026), pages 250–265, Budapest, Hungary. Association for Computational Linguistics.