@inproceedings{cui-etal-2022-generalized,
title = "Generalized Quantifiers as a Source of Error in Multilingual {NLU} Benchmarks",
author = "Cui, Ruixiang and
Hershcovich, Daniel and
S{\o}gaard, Anders",
editor = "Bartolo, Max and
Kirk, Hannah and
Rodriguez, Pedro and
Margatina, Katerina and
Thrush, Tristan and
Jia, Robin and
Stenetorp, Pontus and
Williams, Adina and
Kiela, Douwe",
booktitle = "Proceedings of the First Workshop on Dynamic Adversarial Data Collection",
month = jul,
year = "2022",
address = "Seattle, WA",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2022.dadc-1.7",
doi = "10.18653/v1/2022.dadc-1.7",
pages = "61--61",
abstract = "Logical approaches to representing language have developed and evaluated computational models of quantifier words since the 19th century, but today{'}s NLU models still struggle to capture their semantics. We rely on Generalized Quantifier Theory for language-independent representations of the semantics of quantifier words, to quantify their contribution to the errors of NLU models. We find that quantifiers are pervasive in NLU benchmarks, and their occurrence at test time is associated with performance drops. Multilingual models also exhibit unsatisfying quantifier reasoning abilities, but not necessarily worse for non-English languages. To facilitate directly-targeted probing, we present an adversarial generalized quantifier NLI task (GQNLI) and show that pre-trained language models have a clear lack of robustness in generalized quantifier reasoning.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="cui-etal-2022-generalized">
<titleInfo>
<title>Generalized Quantifiers as a Source of Error in Multilingual NLU Benchmarks</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ruixiang</namePart>
<namePart type="family">Cui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Hershcovich</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anders</namePart>
<namePart type="family">Søgaard</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2022-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Dynamic Adversarial Data Collection</title>
</titleInfo>
<name type="personal">
<namePart type="given">Max</namePart>
<namePart type="family">Bartolo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hannah</namePart>
<namePart type="family">Kirk</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pedro</namePart>
<namePart type="family">Rodriguez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Katerina</namePart>
<namePart type="family">Margatina</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tristan</namePart>
<namePart type="family">Thrush</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Robin</namePart>
<namePart type="family">Jia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pontus</namePart>
<namePart type="family">Stenetorp</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Adina</namePart>
<namePart type="family">Williams</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Douwe</namePart>
<namePart type="family">Kiela</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Seattle, WA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Logical approaches to representing language have developed and evaluated computational models of quantifier words since the 19th century, but today’s NLU models still struggle to capture their semantics. We rely on Generalized Quantifier Theory for language-independent representations of the semantics of quantifier words, to quantify their contribution to the errors of NLU models. We find that quantifiers are pervasive in NLU benchmarks, and their occurrence at test time is associated with performance drops. Multilingual models also exhibit unsatisfying quantifier reasoning abilities, but not necessarily worse for non-English languages. To facilitate directly-targeted probing, we present an adversarial generalized quantifier NLI task (GQNLI) and show that pre-trained language models have a clear lack of robustness in generalized quantifier reasoning.</abstract>
<identifier type="citekey">cui-etal-2022-generalized</identifier>
<identifier type="doi">10.18653/v1/2022.dadc-1.7</identifier>
<location>
<url>https://aclanthology.org/2022.dadc-1.7</url>
</location>
<part>
<date>2022-07</date>
<extent unit="page">
<start>61</start>
<end>61</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Generalized Quantifiers as a Source of Error in Multilingual NLU Benchmarks
%A Cui, Ruixiang
%A Hershcovich, Daniel
%A Søgaard, Anders
%Y Bartolo, Max
%Y Kirk, Hannah
%Y Rodriguez, Pedro
%Y Margatina, Katerina
%Y Thrush, Tristan
%Y Jia, Robin
%Y Stenetorp, Pontus
%Y Williams, Adina
%Y Kiela, Douwe
%S Proceedings of the First Workshop on Dynamic Adversarial Data Collection
%D 2022
%8 July
%I Association for Computational Linguistics
%C Seattle, WA
%F cui-etal-2022-generalized
%X Logical approaches to representing language have developed and evaluated computational models of quantifier words since the 19th century, but today’s NLU models still struggle to capture their semantics. We rely on Generalized Quantifier Theory for language-independent representations of the semantics of quantifier words, to quantify their contribution to the errors of NLU models. We find that quantifiers are pervasive in NLU benchmarks, and their occurrence at test time is associated with performance drops. Multilingual models also exhibit unsatisfying quantifier reasoning abilities, but not necessarily worse for non-English languages. To facilitate directly-targeted probing, we present an adversarial generalized quantifier NLI task (GQNLI) and show that pre-trained language models have a clear lack of robustness in generalized quantifier reasoning.
%R 10.18653/v1/2022.dadc-1.7
%U https://aclanthology.org/2022.dadc-1.7
%U https://doi.org/10.18653/v1/2022.dadc-1.7
%P 61-61
Markdown (Informal)
[Generalized Quantifiers as a Source of Error in Multilingual NLU Benchmarks](https://aclanthology.org/2022.dadc-1.7) (Cui et al., DADC 2022)
ACL