@inproceedings{xu-etal-2021-soft,
title = "Soft Layer Selection with Meta-Learning for Zero-Shot Cross-Lingual Transfer",
author = "Xu, Weijia and
Haider, Batool and
Krone, Jason and
Mansour, Saab",
editor = "Lee, Hung-Yi and
Mohtarami, Mitra and
Li, Shang-Wen and
Jin, Di and
Korpusik, Mandy and
Dong, Shuyan and
Vu, Ngoc Thang and
Hakkani-Tur, Dilek",
booktitle = "Proceedings of the 1st Workshop on Meta Learning and Its Applications to Natural Language Processing",
month = aug,
year = "2021",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2021.metanlp-1.2",
doi = "10.18653/v1/2021.metanlp-1.2",
pages = "11--18",
abstract = "Multilingual pre-trained contextual embedding models (Devlin et al., 2019) have achieved impressive performance on zero-shot cross-lingual transfer tasks. Finding the most effective fine-tuning strategy to fine-tune these models on high-resource languages so that it transfers well to the zero-shot languages is a non-trivial task. In this paper, we propose a novel meta-optimizer to soft-select which layers of the pre-trained model to freeze during fine-tuning. We train the meta-optimizer by simulating the zero-shot transfer scenario. Results on cross-lingual natural language inference show that our approach improves over the simple fine-tuning baseline and X-MAML (Nooralahzadeh et al., 2020).",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="xu-etal-2021-soft">
<titleInfo>
<title>Soft Layer Selection with Meta-Learning for Zero-Shot Cross-Lingual Transfer</title>
</titleInfo>
<name type="personal">
<namePart type="given">Weijia</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Batool</namePart>
<namePart type="family">Haider</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jason</namePart>
<namePart type="family">Krone</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Saab</namePart>
<namePart type="family">Mansour</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2021-08</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Meta Learning and Its Applications to Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mitra</namePart>
<namePart type="family">Mohtarami</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shang-Wen</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Jin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mandy</namePart>
<namePart type="family">Korpusik</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shuyan</namePart>
<namePart type="family">Dong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ngoc</namePart>
<namePart type="given">Thang</namePart>
<namePart type="family">Vu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dilek</namePart>
<namePart type="family">Hakkani-Tur</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Online</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Multilingual pre-trained contextual embedding models (Devlin et al., 2019) have achieved impressive performance on zero-shot cross-lingual transfer tasks. Finding the most effective fine-tuning strategy to fine-tune these models on high-resource languages so that it transfers well to the zero-shot languages is a non-trivial task. In this paper, we propose a novel meta-optimizer to soft-select which layers of the pre-trained model to freeze during fine-tuning. We train the meta-optimizer by simulating the zero-shot transfer scenario. Results on cross-lingual natural language inference show that our approach improves over the simple fine-tuning baseline and X-MAML (Nooralahzadeh et al., 2020).</abstract>
<identifier type="citekey">xu-etal-2021-soft</identifier>
<identifier type="doi">10.18653/v1/2021.metanlp-1.2</identifier>
<location>
<url>https://aclanthology.org/2021.metanlp-1.2</url>
</location>
<part>
<date>2021-08</date>
<extent unit="page">
<start>11</start>
<end>18</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Soft Layer Selection with Meta-Learning for Zero-Shot Cross-Lingual Transfer
%A Xu, Weijia
%A Haider, Batool
%A Krone, Jason
%A Mansour, Saab
%Y Lee, Hung-Yi
%Y Mohtarami, Mitra
%Y Li, Shang-Wen
%Y Jin, Di
%Y Korpusik, Mandy
%Y Dong, Shuyan
%Y Vu, Ngoc Thang
%Y Hakkani-Tur, Dilek
%S Proceedings of the 1st Workshop on Meta Learning and Its Applications to Natural Language Processing
%D 2021
%8 August
%I Association for Computational Linguistics
%C Online
%F xu-etal-2021-soft
%X Multilingual pre-trained contextual embedding models (Devlin et al., 2019) have achieved impressive performance on zero-shot cross-lingual transfer tasks. Finding the most effective fine-tuning strategy to fine-tune these models on high-resource languages so that it transfers well to the zero-shot languages is a non-trivial task. In this paper, we propose a novel meta-optimizer to soft-select which layers of the pre-trained model to freeze during fine-tuning. We train the meta-optimizer by simulating the zero-shot transfer scenario. Results on cross-lingual natural language inference show that our approach improves over the simple fine-tuning baseline and X-MAML (Nooralahzadeh et al., 2020).
%R 10.18653/v1/2021.metanlp-1.2
%U https://aclanthology.org/2021.metanlp-1.2
%U https://doi.org/10.18653/v1/2021.metanlp-1.2
%P 11-18
Markdown (Informal)
[Soft Layer Selection with Meta-Learning for Zero-Shot Cross-Lingual Transfer](https://aclanthology.org/2021.metanlp-1.2) (Xu et al., MetaNLP 2021)
ACL