@inproceedings{romero-diaz-etal-2022-collecting,
title = "Collecting high-quality adversarial data for machine reading comprehension tasks with humans and models in the loop",
author = "Romero Diaz, Damian Y. and
Anio{\l}, Magdalena and
Culnan, John",
editor = "Bartolo, Max and
Kirk, Hannah and
Rodriguez, Pedro and
Margatina, Katerina and
Thrush, Tristan and
Jia, Robin and
Stenetorp, Pontus and
Williams, Adina and
Kiela, Douwe",
booktitle = "Proceedings of the First Workshop on Dynamic Adversarial Data Collection",
month = jul,
year = "2022",
address = "Seattle, WA",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2022.dadc-1.6",
doi = "10.18653/v1/2022.dadc-1.6",
pages = "53--60",
abstract = "We present our experience as annotators in the creation of high-quality, adversarial machine-reading-comprehension data for extractive QA for Task 1 of the First Workshop on Dynamic Adversarial Data Collection (DADC). DADC is an emergent data collection paradigm with both models and humans in the loop. We set up a quasi-experimental annotation design and perform quantitative analyses across groups with different numbers of annotators focusing on successful adversarial attacks, cost analysis, and annotator confidence correlation. We further perform a qualitative analysis of our perceived difficulty of the task given the different topics of the passages in our dataset and conclude with recommendations and suggestions that might be of value to people working on future DADC tasks and related annotation interfaces.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="romero-diaz-etal-2022-collecting">
<titleInfo>
<title>Collecting high-quality adversarial data for machine reading comprehension tasks with humans and models in the loop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Damian</namePart>
<namePart type="given">Y</namePart>
<namePart type="family">Romero Diaz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Magdalena</namePart>
<namePart type="family">Anioł</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">John</namePart>
<namePart type="family">Culnan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2022-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Dynamic Adversarial Data Collection</title>
</titleInfo>
<name type="personal">
<namePart type="given">Max</namePart>
<namePart type="family">Bartolo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hannah</namePart>
<namePart type="family">Kirk</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pedro</namePart>
<namePart type="family">Rodriguez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Katerina</namePart>
<namePart type="family">Margatina</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tristan</namePart>
<namePart type="family">Thrush</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Robin</namePart>
<namePart type="family">Jia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pontus</namePart>
<namePart type="family">Stenetorp</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Adina</namePart>
<namePart type="family">Williams</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Douwe</namePart>
<namePart type="family">Kiela</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Seattle, WA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>We present our experience as annotators in the creation of high-quality, adversarial machine-reading-comprehension data for extractive QA for Task 1 of the First Workshop on Dynamic Adversarial Data Collection (DADC). DADC is an emergent data collection paradigm with both models and humans in the loop. We set up a quasi-experimental annotation design and perform quantitative analyses across groups with different numbers of annotators focusing on successful adversarial attacks, cost analysis, and annotator confidence correlation. We further perform a qualitative analysis of our perceived difficulty of the task given the different topics of the passages in our dataset and conclude with recommendations and suggestions that might be of value to people working on future DADC tasks and related annotation interfaces.</abstract>
<identifier type="citekey">romero-diaz-etal-2022-collecting</identifier>
<identifier type="doi">10.18653/v1/2022.dadc-1.6</identifier>
<location>
<url>https://aclanthology.org/2022.dadc-1.6</url>
</location>
<part>
<date>2022-07</date>
<extent unit="page">
<start>53</start>
<end>60</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Collecting high-quality adversarial data for machine reading comprehension tasks with humans and models in the loop
%A Romero Diaz, Damian Y.
%A Anioł, Magdalena
%A Culnan, John
%Y Bartolo, Max
%Y Kirk, Hannah
%Y Rodriguez, Pedro
%Y Margatina, Katerina
%Y Thrush, Tristan
%Y Jia, Robin
%Y Stenetorp, Pontus
%Y Williams, Adina
%Y Kiela, Douwe
%S Proceedings of the First Workshop on Dynamic Adversarial Data Collection
%D 2022
%8 July
%I Association for Computational Linguistics
%C Seattle, WA
%F romero-diaz-etal-2022-collecting
%X We present our experience as annotators in the creation of high-quality, adversarial machine-reading-comprehension data for extractive QA for Task 1 of the First Workshop on Dynamic Adversarial Data Collection (DADC). DADC is an emergent data collection paradigm with both models and humans in the loop. We set up a quasi-experimental annotation design and perform quantitative analyses across groups with different numbers of annotators focusing on successful adversarial attacks, cost analysis, and annotator confidence correlation. We further perform a qualitative analysis of our perceived difficulty of the task given the different topics of the passages in our dataset and conclude with recommendations and suggestions that might be of value to people working on future DADC tasks and related annotation interfaces.
%R 10.18653/v1/2022.dadc-1.6
%U https://aclanthology.org/2022.dadc-1.6
%U https://doi.org/10.18653/v1/2022.dadc-1.6
%P 53-60
Markdown (Informal)
[Collecting high-quality adversarial data for machine reading comprehension tasks with humans and models in the loop](https://aclanthology.org/2022.dadc-1.6) (Romero Diaz et al., DADC 2022)
ACL