@inproceedings{ravishankar-etal-2019-multilingual,
title = "Multilingual Probing of Deep Pre-Trained Contextual Encoders",
author = {Ravishankar, Vinit and
G{\"o}k{\i}rmak, Memduh and
{\O}vrelid, Lilja and
Velldal, Erik},
editor = {Nivre, Joakim and
Derczynski, Leon and
Ginter, Filip and
Lindi, Bj{\o}rn and
Oepen, Stephan and
S{\o}gaard, Anders and
Tidemann, J{\"o}rg},
booktitle = "Proceedings of the First NLPL Workshop on Deep Learning for Natural Language Processing",
month = sep,
year = "2019",
address = "Turku, Finland",
publisher = {Link{\"o}ping University Electronic Press},
url = "https://aclanthology.org/W19-6205",
pages = "37--47",
abstract = "Encoders that generate representations based on context have, in recent years, benefited from adaptations that allow for pre-training on large text corpora. Earlier work on evaluating fixed-length sentence representations has included the use of {`}probing{'} tasks, that use diagnostic classifiers to attempt to quantify the extent to which these encoders capture specific linguistic phenomena. The principle of probing has also resulted in extended evaluations that include relatively newer word-level pre-trained encoders. We build on probing tasks established in the literature and comprehensively evaluate and analyse {--} from a typological perspective amongst others {--} multilingual variants of existing encoders on probing datasets constructed for 6 non-English languages. Specifically, we probe each layer of a multiple monolingual RNN-based ELMo models, the transformer-based BERT{'}s cased and uncased multilingual variants, and a variant of BERT that uses a cross-lingual modelling scheme (XLM).",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ravishankar-etal-2019-multilingual">
<titleInfo>
<title>Multilingual Probing of Deep Pre-Trained Contextual Encoders</title>
</titleInfo>
<name type="personal">
<namePart type="given">Vinit</namePart>
<namePart type="family">Ravishankar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Memduh</namePart>
<namePart type="family">Gökırmak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lilja</namePart>
<namePart type="family">Øvrelid</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Erik</namePart>
<namePart type="family">Velldal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2019-09</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First NLPL Workshop on Deep Learning for Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Joakim</namePart>
<namePart type="family">Nivre</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leon</namePart>
<namePart type="family">Derczynski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Filip</namePart>
<namePart type="family">Ginter</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bjørn</namePart>
<namePart type="family">Lindi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Stephan</namePart>
<namePart type="family">Oepen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anders</namePart>
<namePart type="family">Søgaard</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jörg</namePart>
<namePart type="family">Tidemann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Linköping University Electronic Press</publisher>
<place>
<placeTerm type="text">Turku, Finland</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Encoders that generate representations based on context have, in recent years, benefited from adaptations that allow for pre-training on large text corpora. Earlier work on evaluating fixed-length sentence representations has included the use of ‘probing’ tasks, that use diagnostic classifiers to attempt to quantify the extent to which these encoders capture specific linguistic phenomena. The principle of probing has also resulted in extended evaluations that include relatively newer word-level pre-trained encoders. We build on probing tasks established in the literature and comprehensively evaluate and analyse – from a typological perspective amongst others – multilingual variants of existing encoders on probing datasets constructed for 6 non-English languages. Specifically, we probe each layer of a multiple monolingual RNN-based ELMo models, the transformer-based BERT’s cased and uncased multilingual variants, and a variant of BERT that uses a cross-lingual modelling scheme (XLM).</abstract>
<identifier type="citekey">ravishankar-etal-2019-multilingual</identifier>
<location>
<url>https://aclanthology.org/W19-6205</url>
</location>
<part>
<date>2019-09</date>
<extent unit="page">
<start>37</start>
<end>47</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Multilingual Probing of Deep Pre-Trained Contextual Encoders
%A Ravishankar, Vinit
%A Gökırmak, Memduh
%A Øvrelid, Lilja
%A Velldal, Erik
%Y Nivre, Joakim
%Y Derczynski, Leon
%Y Ginter, Filip
%Y Lindi, Bjørn
%Y Oepen, Stephan
%Y Søgaard, Anders
%Y Tidemann, Jörg
%S Proceedings of the First NLPL Workshop on Deep Learning for Natural Language Processing
%D 2019
%8 September
%I Linköping University Electronic Press
%C Turku, Finland
%F ravishankar-etal-2019-multilingual
%X Encoders that generate representations based on context have, in recent years, benefited from adaptations that allow for pre-training on large text corpora. Earlier work on evaluating fixed-length sentence representations has included the use of ‘probing’ tasks, that use diagnostic classifiers to attempt to quantify the extent to which these encoders capture specific linguistic phenomena. The principle of probing has also resulted in extended evaluations that include relatively newer word-level pre-trained encoders. We build on probing tasks established in the literature and comprehensively evaluate and analyse – from a typological perspective amongst others – multilingual variants of existing encoders on probing datasets constructed for 6 non-English languages. Specifically, we probe each layer of a multiple monolingual RNN-based ELMo models, the transformer-based BERT’s cased and uncased multilingual variants, and a variant of BERT that uses a cross-lingual modelling scheme (XLM).
%U https://aclanthology.org/W19-6205
%P 37-47
Markdown (Informal)
[Multilingual Probing of Deep Pre-Trained Contextual Encoders](https://aclanthology.org/W19-6205) (Ravishankar et al., NoDaLiDa 2019)
ACL