@inproceedings{jacob-etal-2026-disagreement,
title = "When Disagreement Meets Noise: Noise Robust Annotator Embeddings for Subjective {NLP}",
author = "Jacob, Kevin and
Ghazvinian, Parsa and
Jinadu, Uthman and
Yang, Nicole Tianjiao and
Ding, Yi",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2078/",
pages = "44254--44271",
ISBN = "979-8-89176-462-0",
abstract = "Subjective NLP tasks such as sentiment analysis and hate speech classification often involve inherent annotator disagreement, reflecting diverse perspectives shaped by annotators' lived experiences. Although conventional approaches resolve disagreement through majority voting or aggregation, these methods risk erasing valuable nuances and minority viewpoints. Recent embedding-based and multitask models have advanced the modeling of annotator-specific judgments, yet their robustness to annotation noise remains underexplored. In this work, we systematically investigate how state-of-the-art multi-annotator learning models perform in the presence of noisy labels, and find that embedding-based approaches are more robust than aggregation and multitask baselines on datasets with many sparse annotators. Furthermore, we propose a new methodology, Noise-Robust Annotator Embedding (NRA-Embed), which integrates Robust InfoNCE (RINCE) contrastive loss to enhance models' robustness under noisy annotation conditions. Through extensive experiments, we demonstrate that NRA-Embed effectively captures subjective disagreement while maintaining robustness to noise, achieving performance that is comparable to state-of-the-art in clean settings while providing numerically greater robustness under noise."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="jacob-etal-2026-disagreement">
<titleInfo>
<title>When Disagreement Meets Noise: Noise Robust Annotator Embeddings for Subjective NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kevin</namePart>
<namePart type="family">Jacob</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Parsa</namePart>
<namePart type="family">Ghazvinian</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Uthman</namePart>
<namePart type="family">Jinadu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicole</namePart>
<namePart type="given">Tianjiao</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="family">Ding</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Subjective NLP tasks such as sentiment analysis and hate speech classification often involve inherent annotator disagreement, reflecting diverse perspectives shaped by annotators’ lived experiences. Although conventional approaches resolve disagreement through majority voting or aggregation, these methods risk erasing valuable nuances and minority viewpoints. Recent embedding-based and multitask models have advanced the modeling of annotator-specific judgments, yet their robustness to annotation noise remains underexplored. In this work, we systematically investigate how state-of-the-art multi-annotator learning models perform in the presence of noisy labels, and find that embedding-based approaches are more robust than aggregation and multitask baselines on datasets with many sparse annotators. Furthermore, we propose a new methodology, Noise-Robust Annotator Embedding (NRA-Embed), which integrates Robust InfoNCE (RINCE) contrastive loss to enhance models’ robustness under noisy annotation conditions. Through extensive experiments, we demonstrate that NRA-Embed effectively captures subjective disagreement while maintaining robustness to noise, achieving performance that is comparable to state-of-the-art in clean settings while providing numerically greater robustness under noise.</abstract>
<identifier type="citekey">jacob-etal-2026-disagreement</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2078/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>44254</start>
<end>44271</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Disagreement Meets Noise: Noise Robust Annotator Embeddings for Subjective NLP
%A Jacob, Kevin
%A Ghazvinian, Parsa
%A Jinadu, Uthman
%A Yang, Nicole Tianjiao
%A Ding, Yi
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F jacob-etal-2026-disagreement
%X Subjective NLP tasks such as sentiment analysis and hate speech classification often involve inherent annotator disagreement, reflecting diverse perspectives shaped by annotators’ lived experiences. Although conventional approaches resolve disagreement through majority voting or aggregation, these methods risk erasing valuable nuances and minority viewpoints. Recent embedding-based and multitask models have advanced the modeling of annotator-specific judgments, yet their robustness to annotation noise remains underexplored. In this work, we systematically investigate how state-of-the-art multi-annotator learning models perform in the presence of noisy labels, and find that embedding-based approaches are more robust than aggregation and multitask baselines on datasets with many sparse annotators. Furthermore, we propose a new methodology, Noise-Robust Annotator Embedding (NRA-Embed), which integrates Robust InfoNCE (RINCE) contrastive loss to enhance models’ robustness under noisy annotation conditions. Through extensive experiments, we demonstrate that NRA-Embed effectively captures subjective disagreement while maintaining robustness to noise, achieving performance that is comparable to state-of-the-art in clean settings while providing numerically greater robustness under noise.
%U https://aclanthology.org/2026.findings-emnlp.2078/
%P 44254-44271
Markdown (Informal)
[When Disagreement Meets Noise: Noise Robust Annotator Embeddings for Subjective NLP](https://aclanthology.org/2026.findings-emnlp.2078/) (Jacob et al., Findings 2026)
ACL