@article{belz-2022-metrological,
title = "A Metrological Perspective on Reproducibility in {NLP}*",
author = "Belz, Anya",
journal = "Computational Linguistics",
volume = "48",
number = "4",
month = dec,
year = "2022",
address = "Cambridge, MA",
publisher = "MIT Press",
url = "https://aclanthology.org/2022.cl-4.21",
doi = "10.1162/coli_a_00448",
pages = "1125--1135",
abstract = "Reproducibility has become an increasingly debated topic in NLP and ML over recent years, but so far, no commonly accepted definitions of even basic terms or concepts have emerged. The range of different definitions proposed within NLP/ML not only do not agree with each other, they are also not aligned with standard scientific definitions. This article examines the standard definitions of repeatability and reproducibility provided by the meta-science of metrology, and explores what they imply in terms of how to assess reproducibility, and what adopting them would mean for reproducibility assessment in NLP/ML. It turns out the standard definitions lead directly to a method for assessing reproducibility in quantified terms that renders results from reproduction studies comparable across multiple reproductions of the same original study, as well as reproductions of different original studies. The article considers where this method sits in relation to other aspects of NLP work one might wish to assess in the context of reproducibility.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="belz-2022-metrological">
<titleInfo>
<title>A Metrological Perspective on Reproducibility in NLP*</title>
</titleInfo>
<name type="personal">
<namePart type="given">Anya</namePart>
<namePart type="family">Belz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2022-12</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<genre authority="bibutilsgt">journal article</genre>
<relatedItem type="host">
<titleInfo>
<title>Computational Linguistics</title>
</titleInfo>
<originInfo>
<issuance>continuing</issuance>
<publisher>MIT Press</publisher>
<place>
<placeTerm type="text">Cambridge, MA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">periodical</genre>
<genre authority="bibutilsgt">academic journal</genre>
</relatedItem>
<abstract>Reproducibility has become an increasingly debated topic in NLP and ML over recent years, but so far, no commonly accepted definitions of even basic terms or concepts have emerged. The range of different definitions proposed within NLP/ML not only do not agree with each other, they are also not aligned with standard scientific definitions. This article examines the standard definitions of repeatability and reproducibility provided by the meta-science of metrology, and explores what they imply in terms of how to assess reproducibility, and what adopting them would mean for reproducibility assessment in NLP/ML. It turns out the standard definitions lead directly to a method for assessing reproducibility in quantified terms that renders results from reproduction studies comparable across multiple reproductions of the same original study, as well as reproductions of different original studies. The article considers where this method sits in relation to other aspects of NLP work one might wish to assess in the context of reproducibility.</abstract>
<identifier type="citekey">belz-2022-metrological</identifier>
<identifier type="doi">10.1162/coli_a_00448</identifier>
<location>
<url>https://aclanthology.org/2022.cl-4.21</url>
</location>
<part>
<date>2022-12</date>
<detail type="volume"><number>48</number></detail>
<detail type="issue"><number>4</number></detail>
<extent unit="page">
<start>1125</start>
<end>1135</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Journal Article
%T A Metrological Perspective on Reproducibility in NLP*
%A Belz, Anya
%J Computational Linguistics
%D 2022
%8 December
%V 48
%N 4
%I MIT Press
%C Cambridge, MA
%F belz-2022-metrological
%X Reproducibility has become an increasingly debated topic in NLP and ML over recent years, but so far, no commonly accepted definitions of even basic terms or concepts have emerged. The range of different definitions proposed within NLP/ML not only do not agree with each other, they are also not aligned with standard scientific definitions. This article examines the standard definitions of repeatability and reproducibility provided by the meta-science of metrology, and explores what they imply in terms of how to assess reproducibility, and what adopting them would mean for reproducibility assessment in NLP/ML. It turns out the standard definitions lead directly to a method for assessing reproducibility in quantified terms that renders results from reproduction studies comparable across multiple reproductions of the same original study, as well as reproductions of different original studies. The article considers where this method sits in relation to other aspects of NLP work one might wish to assess in the context of reproducibility.
%R 10.1162/coli_a_00448
%U https://aclanthology.org/2022.cl-4.21
%U https://doi.org/10.1162/coli_a_00448
%P 1125-1135
Markdown (Informal)
[A Metrological Perspective on Reproducibility in NLP*](https://aclanthology.org/2022.cl-4.21) (Belz, CL 2022)
ACL