@inproceedings{pikuliak-simko-2022-average,
title = "Average Is Not Enough: Caveats of Multilingual Evaluation",
author = "Pikuliak, Mat{\'u}{\v{s}} and
Simko, Marian",
editor = {Ataman, Duygu and
Gonen, Hila and
Ruder, Sebastian and
Firat, Orhan and
G{\"u}l Sahin, G{\"o}zde and
Mirzakhalov, Jamshidbek},
booktitle = "Proceedings of the The 2nd Workshop on Multi-lingual Representation Learning (MRL)",
month = dec,
year = "2022",
address = "Abu Dhabi, United Arab Emirates (Hybrid)",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2022.mrl-1.13",
doi = "10.18653/v1/2022.mrl-1.13",
pages = "125--133",
abstract = "This position paper discusses the problem of multilingual evaluation. Using simple statistics, such as average language performance, might inject linguistic biases in favor of dominant language families into evaluation methodology. We argue that a qualitative analysis informed by comparative linguistics is needed for multilingual results to detect this kind of bias. We show in our case study that results in published works can indeed be linguistically biased and we demonstrate that visualization based on URIEL typological database can detect it.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="pikuliak-simko-2022-average">
<titleInfo>
<title>Average Is Not Enough: Caveats of Multilingual Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Matúš</namePart>
<namePart type="family">Pikuliak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marian</namePart>
<namePart type="family">Simko</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2022-12</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the The 2nd Workshop on Multi-lingual Representation Learning (MRL)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Duygu</namePart>
<namePart type="family">Ataman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hila</namePart>
<namePart type="family">Gonen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sebastian</namePart>
<namePart type="family">Ruder</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Orhan</namePart>
<namePart type="family">Firat</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gözde</namePart>
<namePart type="family">Gül Sahin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jamshidbek</namePart>
<namePart type="family">Mirzakhalov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Abu Dhabi, United Arab Emirates (Hybrid)</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>This position paper discusses the problem of multilingual evaluation. Using simple statistics, such as average language performance, might inject linguistic biases in favor of dominant language families into evaluation methodology. We argue that a qualitative analysis informed by comparative linguistics is needed for multilingual results to detect this kind of bias. We show in our case study that results in published works can indeed be linguistically biased and we demonstrate that visualization based on URIEL typological database can detect it.</abstract>
<identifier type="citekey">pikuliak-simko-2022-average</identifier>
<identifier type="doi">10.18653/v1/2022.mrl-1.13</identifier>
<location>
<url>https://aclanthology.org/2022.mrl-1.13</url>
</location>
<part>
<date>2022-12</date>
<extent unit="page">
<start>125</start>
<end>133</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Average Is Not Enough: Caveats of Multilingual Evaluation
%A Pikuliak, Matúš
%A Simko, Marian
%Y Ataman, Duygu
%Y Gonen, Hila
%Y Ruder, Sebastian
%Y Firat, Orhan
%Y Gül Sahin, Gözde
%Y Mirzakhalov, Jamshidbek
%S Proceedings of the The 2nd Workshop on Multi-lingual Representation Learning (MRL)
%D 2022
%8 December
%I Association for Computational Linguistics
%C Abu Dhabi, United Arab Emirates (Hybrid)
%F pikuliak-simko-2022-average
%X This position paper discusses the problem of multilingual evaluation. Using simple statistics, such as average language performance, might inject linguistic biases in favor of dominant language families into evaluation methodology. We argue that a qualitative analysis informed by comparative linguistics is needed for multilingual results to detect this kind of bias. We show in our case study that results in published works can indeed be linguistically biased and we demonstrate that visualization based on URIEL typological database can detect it.
%R 10.18653/v1/2022.mrl-1.13
%U https://aclanthology.org/2022.mrl-1.13
%U https://doi.org/10.18653/v1/2022.mrl-1.13
%P 125-133
Markdown (Informal)
[Average Is Not Enough: Caveats of Multilingual Evaluation](https://aclanthology.org/2022.mrl-1.13) (Pikuliak & Simko, MRL 2022)
ACL