@inproceedings{andrade-etal-2026-llms,
title = "{LLM}s as Assistants for Data Annotation: Addressing Disagreement and Supporting Expert Processes",
author = "Andrade, Mark and
Heffernan, Bl{\'a}ith{\'i}n and
Walsh, Abigail and
Castilho, Sheila",
editor = "Morger, Felix and
Ilinykh, Nikolai and
Scalvini, Barbara and
Dobnik, Simon and
Dann{\'e}lls, Dana",
booktitle = "Proceedings of the Fourth Workshop on the Role of Resources in the Age of Large Language Models ({RESOURCEFUL} 2026)",
month = may,
year = "2026",
address = "Palma, Mallorca (Spain)",
publisher = "ELRA Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.resourceful-4.7/",
doi = "10.63317/4gq9fnt5umo3",
pages = "62--72",
abstract = "This paper investigates the potential of Large Language Models to assist human annotation pipelines, with a particular focus on supporting the development of expert-informed annotation guidelines for document-level content categorisation. We present three experiments exploring distinct roles for LLMs in annotation: as annotators, as domain experts assisting in disagreement resolution, and as analysts of annotator discussions. Using GPT-4.5 and Claude Sonnet 4, we evaluate LLM-generated annotation guidelines for a document-level classification tasks in terms of coverage, applicability, and usefulness. Preliminary results are mixed-to-positive, with evidence that LLMs can provide useful support across different stages of the annotation pipeline, particularly when supplied with rich contextual information such as prior human annotations and annotator discussions. However, their effectiveness remains sensitive to prompting strategies and input configuration."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="andrade-etal-2026-llms">
<titleInfo>
<title>LLMs as Assistants for Data Annotation: Addressing Disagreement and Supporting Expert Processes</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mark</namePart>
<namePart type="family">Andrade</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bláithín</namePart>
<namePart type="family">Heffernan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abigail</namePart>
<namePart type="family">Walsh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sheila</namePart>
<namePart type="family">Castilho</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Fourth Workshop on the Role of Resources in the Age of Large Language Models (RESOURCEFUL 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Felix</namePart>
<namePart type="family">Morger</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikolai</namePart>
<namePart type="family">Ilinykh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Scalvini</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Dobnik</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dana</namePart>
<namePart type="family">Dannélls</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma, Mallorca (Spain)</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>This paper investigates the potential of Large Language Models to assist human annotation pipelines, with a particular focus on supporting the development of expert-informed annotation guidelines for document-level content categorisation. We present three experiments exploring distinct roles for LLMs in annotation: as annotators, as domain experts assisting in disagreement resolution, and as analysts of annotator discussions. Using GPT-4.5 and Claude Sonnet 4, we evaluate LLM-generated annotation guidelines for a document-level classification tasks in terms of coverage, applicability, and usefulness. Preliminary results are mixed-to-positive, with evidence that LLMs can provide useful support across different stages of the annotation pipeline, particularly when supplied with rich contextual information such as prior human annotations and annotator discussions. However, their effectiveness remains sensitive to prompting strategies and input configuration.</abstract>
<identifier type="citekey">andrade-etal-2026-llms</identifier>
<identifier type="doi">10.63317/4gq9fnt5umo3</identifier>
<location>
<url>https://aclanthology.org/2026.resourceful-4.7/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>62</start>
<end>72</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T LLMs as Assistants for Data Annotation: Addressing Disagreement and Supporting Expert Processes
%A Andrade, Mark
%A Heffernan, Bláithín
%A Walsh, Abigail
%A Castilho, Sheila
%Y Morger, Felix
%Y Ilinykh, Nikolai
%Y Scalvini, Barbara
%Y Dobnik, Simon
%Y Dannélls, Dana
%S Proceedings of the Fourth Workshop on the Role of Resources in the Age of Large Language Models (RESOURCEFUL 2026)
%D 2026
%8 May
%I ELRA Language Resources Association (ELRA)
%C Palma, Mallorca (Spain)
%F andrade-etal-2026-llms
%X This paper investigates the potential of Large Language Models to assist human annotation pipelines, with a particular focus on supporting the development of expert-informed annotation guidelines for document-level content categorisation. We present three experiments exploring distinct roles for LLMs in annotation: as annotators, as domain experts assisting in disagreement resolution, and as analysts of annotator discussions. Using GPT-4.5 and Claude Sonnet 4, we evaluate LLM-generated annotation guidelines for a document-level classification tasks in terms of coverage, applicability, and usefulness. Preliminary results are mixed-to-positive, with evidence that LLMs can provide useful support across different stages of the annotation pipeline, particularly when supplied with rich contextual information such as prior human annotations and annotator discussions. However, their effectiveness remains sensitive to prompting strategies and input configuration.
%R 10.63317/4gq9fnt5umo3
%U https://aclanthology.org/2026.resourceful-4.7/
%U https://doi.org/10.63317/4gq9fnt5umo3
%P 62-72
Markdown (Informal)
[LLMs as Assistants for Data Annotation: Addressing Disagreement and Supporting Expert Processes](https://aclanthology.org/2026.resourceful-4.7/) (Andrade et al., RESOURCEFUL 2026)
ACL