@inproceedings{gruber-etal-2024-labels,
title = "More Labels or Cases? Assessing Label Variation in Natural Language Inference",
author = {Gruber, Cornelia and
Hechinger, Katharina and
Assenmacher, Matthias and
Kauermann, G{\"o}ran and
Plank, Barbara},
editor = "Pyatkin, Valentina and
Fried, Daniel and
Stengel-Eskin, Elias and
Stengel-Eskin, Elias and
Liu, Alisa and
Pezzelle, Sandro",
booktitle = "Proceedings of the Third Workshop on Understanding Implicit and Underspecified Language",
month = mar,
year = "2024",
address = "Malta",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2024.unimplicit-1.2",
pages = "22--32",
abstract = "In this work, we analyze the uncertainty that is inherently present in the labels used for supervised machine learning in natural language inference (NLI). In cases where multiple annotations per instance are available, neither the majority vote nor the frequency of individual class votes is a trustworthy representation of the labeling uncertainty. We propose modeling the votes via a Bayesian mixture model to recover the data-generating process, i.e., the {``}true{''} latent classes, and thus gain insight into the class variations. This will enable a better understanding of the confusion happening during the annotation process. We also assess the stability of the proposed estimation procedure by systematically varying the numbers of i) instances and ii) labels. Thereby, we observe that few instances with many labels can predict the latent class borders reasonably well, while the estimation fails for many instances with only a few labels. This leads us to conclude that multiple labels are a crucial building block for properly analyzing label uncertainty.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="gruber-etal-2024-labels">
<titleInfo>
<title>More Labels or Cases? Assessing Label Variation in Natural Language Inference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Cornelia</namePart>
<namePart type="family">Gruber</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Katharina</namePart>
<namePart type="family">Hechinger</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Matthias</namePart>
<namePart type="family">Assenmacher</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Göran</namePart>
<namePart type="family">Kauermann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2024-03</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Third Workshop on Understanding Implicit and Underspecified Language</title>
</titleInfo>
<name type="personal">
<namePart type="given">Valentina</namePart>
<namePart type="family">Pyatkin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Fried</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elias</namePart>
<namePart type="family">Stengel-Eskin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alisa</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sandro</namePart>
<namePart type="family">Pezzelle</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Malta</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>In this work, we analyze the uncertainty that is inherently present in the labels used for supervised machine learning in natural language inference (NLI). In cases where multiple annotations per instance are available, neither the majority vote nor the frequency of individual class votes is a trustworthy representation of the labeling uncertainty. We propose modeling the votes via a Bayesian mixture model to recover the data-generating process, i.e., the “true” latent classes, and thus gain insight into the class variations. This will enable a better understanding of the confusion happening during the annotation process. We also assess the stability of the proposed estimation procedure by systematically varying the numbers of i) instances and ii) labels. Thereby, we observe that few instances with many labels can predict the latent class borders reasonably well, while the estimation fails for many instances with only a few labels. This leads us to conclude that multiple labels are a crucial building block for properly analyzing label uncertainty.</abstract>
<identifier type="citekey">gruber-etal-2024-labels</identifier>
<location>
<url>https://aclanthology.org/2024.unimplicit-1.2</url>
</location>
<part>
<date>2024-03</date>
<extent unit="page">
<start>22</start>
<end>32</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T More Labels or Cases? Assessing Label Variation in Natural Language Inference
%A Gruber, Cornelia
%A Hechinger, Katharina
%A Assenmacher, Matthias
%A Kauermann, Göran
%A Plank, Barbara
%Y Pyatkin, Valentina
%Y Fried, Daniel
%Y Stengel-Eskin, Elias
%Y Liu, Alisa
%Y Pezzelle, Sandro
%S Proceedings of the Third Workshop on Understanding Implicit and Underspecified Language
%D 2024
%8 March
%I Association for Computational Linguistics
%C Malta
%F gruber-etal-2024-labels
%X In this work, we analyze the uncertainty that is inherently present in the labels used for supervised machine learning in natural language inference (NLI). In cases where multiple annotations per instance are available, neither the majority vote nor the frequency of individual class votes is a trustworthy representation of the labeling uncertainty. We propose modeling the votes via a Bayesian mixture model to recover the data-generating process, i.e., the “true” latent classes, and thus gain insight into the class variations. This will enable a better understanding of the confusion happening during the annotation process. We also assess the stability of the proposed estimation procedure by systematically varying the numbers of i) instances and ii) labels. Thereby, we observe that few instances with many labels can predict the latent class borders reasonably well, while the estimation fails for many instances with only a few labels. This leads us to conclude that multiple labels are a crucial building block for properly analyzing label uncertainty.
%U https://aclanthology.org/2024.unimplicit-1.2
%P 22-32
Markdown (Informal)
[More Labels or Cases? Assessing Label Variation in Natural Language Inference](https://aclanthology.org/2024.unimplicit-1.2) (Gruber et al., unimplicit-WS 2024)
ACL