@inproceedings{perez-montero-etal-2026-noisy,
title = "How Noisy is Automatic Annotation? Characterizing Label Noise from Generative Models in Multi-Dimensional {S}panish Subjectivity Annotation",
author = "P{\'e}rez-Montero, Alba and
Lloret, Elena and
Moreda Pozo, Paloma",
editor = "Bak, JinYeong and
van der Goot, Rob and
Jang, Hyeju and
Buaphet, Weerayut and
Ramponi, Alan and
Xu, Wei and
Ritter, Alan",
booktitle = "Proceedings of the 11th Workshop on Natural User-generated Text",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.wnut-1.10/",
pages = "141--153",
ISBN = "979-8-89176-470-5",
abstract = "Although human variability has traditionally been the primary source of annotation noise, the increasing use of Large Language Models (LLMs) for automated labeling is shifting this landscape toward systematic, machine-generated label noise. This challenge is particularly severe in Subjectivity Detection (SD), where the behavior of automated errors remains largely unexamined. This study characterizes the extent and linguistic nature of automated annotation noise in Spanish SD. Errors are evaluated as omissions (under-generation) or additions (over-generation) across six subjectivity dimensions: Doubt, Certainty, Specificity, Non-specificity, Distancing, and Participation. Using an expert-annotated human gold standard as a reference, a traditional machine learning control is benchmarked against three state-of-the-art LLM families (GPT, Gemini, and Llama). The results reveal a clear architectural divide: while the traditional ML framework predominantly generates omission errors (63.99{\%}), only detecting markers it was explicitly programmed to recognize, LLMs like Gemini and Llama lean heavily toward addition errors (exceeding 51{\%}), guessing and hallucinating markers instead of leaving uncertain cases unlabeled. Although Gemini minimizes overall noise, achieving F1-scores up to 0.96 in specific dimensions, Certainty remains highly susceptible to errors, showing a 53.96{\%} error rate in the traditional ML control. Ultimately, this diagnostic evaluation of LLMs as annotators in this specific subjective task reveals that automated systems introduce systematic label noise, demonstrating that human-in-the-loop participation is indispensable to resolve these errors and ensure reliable dataset curation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="perez-montero-etal-2026-noisy">
<titleInfo>
<title>How Noisy is Automatic Annotation? Characterizing Label Noise from Generative Models in Multi-Dimensional Spanish Subjectivity Annotation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Alba</namePart>
<namePart type="family">Pérez-Montero</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elena</namePart>
<namePart type="family">Lloret</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Paloma</namePart>
<namePart type="family">Moreda Pozo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 11th Workshop on Natural User-generated Text</title>
</titleInfo>
<name type="personal">
<namePart type="given">JinYeong</namePart>
<namePart type="family">Bak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rob</namePart>
<namePart type="family">van der Goot</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hyeju</namePart>
<namePart type="family">Jang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weerayut</namePart>
<namePart type="family">Buaphet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Ramponi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Ritter</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-470-5</identifier>
</relatedItem>
<abstract>Although human variability has traditionally been the primary source of annotation noise, the increasing use of Large Language Models (LLMs) for automated labeling is shifting this landscape toward systematic, machine-generated label noise. This challenge is particularly severe in Subjectivity Detection (SD), where the behavior of automated errors remains largely unexamined. This study characterizes the extent and linguistic nature of automated annotation noise in Spanish SD. Errors are evaluated as omissions (under-generation) or additions (over-generation) across six subjectivity dimensions: Doubt, Certainty, Specificity, Non-specificity, Distancing, and Participation. Using an expert-annotated human gold standard as a reference, a traditional machine learning control is benchmarked against three state-of-the-art LLM families (GPT, Gemini, and Llama). The results reveal a clear architectural divide: while the traditional ML framework predominantly generates omission errors (63.99%), only detecting markers it was explicitly programmed to recognize, LLMs like Gemini and Llama lean heavily toward addition errors (exceeding 51%), guessing and hallucinating markers instead of leaving uncertain cases unlabeled. Although Gemini minimizes overall noise, achieving F1-scores up to 0.96 in specific dimensions, Certainty remains highly susceptible to errors, showing a 53.96% error rate in the traditional ML control. Ultimately, this diagnostic evaluation of LLMs as annotators in this specific subjective task reveals that automated systems introduce systematic label noise, demonstrating that human-in-the-loop participation is indispensable to resolve these errors and ensure reliable dataset curation.</abstract>
<identifier type="citekey">perez-montero-etal-2026-noisy</identifier>
<location>
<url>https://aclanthology.org/2026.wnut-1.10/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>141</start>
<end>153</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T How Noisy is Automatic Annotation? Characterizing Label Noise from Generative Models in Multi-Dimensional Spanish Subjectivity Annotation
%A Pérez-Montero, Alba
%A Lloret, Elena
%A Moreda Pozo, Paloma
%Y Bak, JinYeong
%Y van der Goot, Rob
%Y Jang, Hyeju
%Y Buaphet, Weerayut
%Y Ramponi, Alan
%Y Xu, Wei
%Y Ritter, Alan
%S Proceedings of the 11th Workshop on Natural User-generated Text
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-470-5
%F perez-montero-etal-2026-noisy
%X Although human variability has traditionally been the primary source of annotation noise, the increasing use of Large Language Models (LLMs) for automated labeling is shifting this landscape toward systematic, machine-generated label noise. This challenge is particularly severe in Subjectivity Detection (SD), where the behavior of automated errors remains largely unexamined. This study characterizes the extent and linguistic nature of automated annotation noise in Spanish SD. Errors are evaluated as omissions (under-generation) or additions (over-generation) across six subjectivity dimensions: Doubt, Certainty, Specificity, Non-specificity, Distancing, and Participation. Using an expert-annotated human gold standard as a reference, a traditional machine learning control is benchmarked against three state-of-the-art LLM families (GPT, Gemini, and Llama). The results reveal a clear architectural divide: while the traditional ML framework predominantly generates omission errors (63.99%), only detecting markers it was explicitly programmed to recognize, LLMs like Gemini and Llama lean heavily toward addition errors (exceeding 51%), guessing and hallucinating markers instead of leaving uncertain cases unlabeled. Although Gemini minimizes overall noise, achieving F1-scores up to 0.96 in specific dimensions, Certainty remains highly susceptible to errors, showing a 53.96% error rate in the traditional ML control. Ultimately, this diagnostic evaluation of LLMs as annotators in this specific subjective task reveals that automated systems introduce systematic label noise, demonstrating that human-in-the-loop participation is indispensable to resolve these errors and ensure reliable dataset curation.
%U https://aclanthology.org/2026.wnut-1.10/
%P 141-153
Markdown (Informal)
[How Noisy is Automatic Annotation? Characterizing Label Noise from Generative Models in Multi-Dimensional Spanish Subjectivity Annotation](https://aclanthology.org/2026.wnut-1.10/) (Pérez-Montero et al., WNUT 2026)
ACL