@inproceedings{lavee-etal-2019-crowd,
title = "Crowd-sourcing annotation of complex {NLU} tasks: A case study of argumentative content annotation",
author = "Lavee, Tamar and
Kotlerman, Lili and
Orbach, Matan and
Bilu, Yonatan and
Jacovi, Michal and
Aharonov, Ranit and
Slonim, Noam",
editor = "Paun, Silviu and
Hovy, Dirk",
booktitle = "Proceedings of the First Workshop on Aggregating and Analysing Crowdsourced Annotations for NLP",
month = nov,
year = "2019",
address = "Hong Kong",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/D19-5905",
doi = "10.18653/v1/D19-5905",
pages = "29--38",
abstract = "Recent advancements in machine reading and listening comprehension involve the annotation of long texts. Such tasks are typically time consuming, making crowd-annotations an attractive solution, yet their complexity often makes such a solution unfeasible. In particular, a major concern is that crowd annotators may be tempted to skim through long texts, and answer questions without reading thoroughly. We present a case study of adapting this type of task to the crowd. The task is to identify claims in a several minute long debate speech. We show that sentence-by-sentence annotation does not scale and that labeling only a subset of sentences is insufficient. Instead, we propose a scheme for effectively performing the full, complex task with crowd annotators, allowing the collection of large scale annotated datasets. We believe that the encountered challenges and pitfalls, as well as lessons learned, are relevant in general when collecting data for large scale natural language understanding (NLU) tasks.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lavee-etal-2019-crowd">
<titleInfo>
<title>Crowd-sourcing annotation of complex NLU tasks: A case study of argumentative content annotation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Tamar</namePart>
<namePart type="family">Lavee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lili</namePart>
<namePart type="family">Kotlerman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Matan</namePart>
<namePart type="family">Orbach</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yonatan</namePart>
<namePart type="family">Bilu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Michal</namePart>
<namePart type="family">Jacovi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ranit</namePart>
<namePart type="family">Aharonov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Noam</namePart>
<namePart type="family">Slonim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2019-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Aggregating and Analysing Crowdsourced Annotations for NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Silviu</namePart>
<namePart type="family">Paun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dirk</namePart>
<namePart type="family">Hovy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hong Kong</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Recent advancements in machine reading and listening comprehension involve the annotation of long texts. Such tasks are typically time consuming, making crowd-annotations an attractive solution, yet their complexity often makes such a solution unfeasible. In particular, a major concern is that crowd annotators may be tempted to skim through long texts, and answer questions without reading thoroughly. We present a case study of adapting this type of task to the crowd. The task is to identify claims in a several minute long debate speech. We show that sentence-by-sentence annotation does not scale and that labeling only a subset of sentences is insufficient. Instead, we propose a scheme for effectively performing the full, complex task with crowd annotators, allowing the collection of large scale annotated datasets. We believe that the encountered challenges and pitfalls, as well as lessons learned, are relevant in general when collecting data for large scale natural language understanding (NLU) tasks.</abstract>
<identifier type="citekey">lavee-etal-2019-crowd</identifier>
<identifier type="doi">10.18653/v1/D19-5905</identifier>
<location>
<url>https://aclanthology.org/D19-5905</url>
</location>
<part>
<date>2019-11</date>
<extent unit="page">
<start>29</start>
<end>38</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Crowd-sourcing annotation of complex NLU tasks: A case study of argumentative content annotation
%A Lavee, Tamar
%A Kotlerman, Lili
%A Orbach, Matan
%A Bilu, Yonatan
%A Jacovi, Michal
%A Aharonov, Ranit
%A Slonim, Noam
%Y Paun, Silviu
%Y Hovy, Dirk
%S Proceedings of the First Workshop on Aggregating and Analysing Crowdsourced Annotations for NLP
%D 2019
%8 November
%I Association for Computational Linguistics
%C Hong Kong
%F lavee-etal-2019-crowd
%X Recent advancements in machine reading and listening comprehension involve the annotation of long texts. Such tasks are typically time consuming, making crowd-annotations an attractive solution, yet their complexity often makes such a solution unfeasible. In particular, a major concern is that crowd annotators may be tempted to skim through long texts, and answer questions without reading thoroughly. We present a case study of adapting this type of task to the crowd. The task is to identify claims in a several minute long debate speech. We show that sentence-by-sentence annotation does not scale and that labeling only a subset of sentences is insufficient. Instead, we propose a scheme for effectively performing the full, complex task with crowd annotators, allowing the collection of large scale annotated datasets. We believe that the encountered challenges and pitfalls, as well as lessons learned, are relevant in general when collecting data for large scale natural language understanding (NLU) tasks.
%R 10.18653/v1/D19-5905
%U https://aclanthology.org/D19-5905
%U https://doi.org/10.18653/v1/D19-5905
%P 29-38
Markdown (Informal)
[Crowd-sourcing annotation of complex NLU tasks: A case study of argumentative content annotation](https://aclanthology.org/D19-5905) (Lavee et al., 2019)
ACL