@inproceedings{wold-2022-effectiveness,
title = "The Effectiveness of Masked Language Modeling and Adapters for Factual Knowledge Injection",
author = "Wold, Sondre",
editor = "Ustalov, Dmitry and
Gao, Yanjun and
Panchenko, Alexander and
Valentino, Marco and
Thayaparan, Mokanarangan and
Nguyen, Thien Huu and
Penn, Gerald and
Ramesh, Arti and
Jana, Abhik",
booktitle = "Proceedings of TextGraphs-16: Graph-based Methods for Natural Language Processing",
month = oct,
year = "2022",
address = "Gyeongju, Republic of Korea",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2022.textgraphs-1.6",
pages = "54--59",
abstract = "This paper studies the problem of injecting factual knowledge into large pre-trained language models. We train adapter modules on parts of the ConceptNet knowledge graph using the masked language modeling objective and evaluate the success of the method by a series of probing experiments on the LAMA probe. Mean P@K curves for different configurations indicate that the technique is effective, increasing the performance on sub-sets of the LAMA probe for large values of k by adding as little as 2.1{\%} additional parameters to the original models.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wold-2022-effectiveness">
<titleInfo>
<title>The Effectiveness of Masked Language Modeling and Adapters for Factual Knowledge Injection</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sondre</namePart>
<namePart type="family">Wold</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2022-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of TextGraphs-16: Graph-based Methods for Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dmitry</namePart>
<namePart type="family">Ustalov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yanjun</namePart>
<namePart type="family">Gao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexander</namePart>
<namePart type="family">Panchenko</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marco</namePart>
<namePart type="family">Valentino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mokanarangan</namePart>
<namePart type="family">Thayaparan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thien</namePart>
<namePart type="given">Huu</namePart>
<namePart type="family">Nguyen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gerald</namePart>
<namePart type="family">Penn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Arti</namePart>
<namePart type="family">Ramesh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abhik</namePart>
<namePart type="family">Jana</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Gyeongju, Republic of Korea</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>This paper studies the problem of injecting factual knowledge into large pre-trained language models. We train adapter modules on parts of the ConceptNet knowledge graph using the masked language modeling objective and evaluate the success of the method by a series of probing experiments on the LAMA probe. Mean P@K curves for different configurations indicate that the technique is effective, increasing the performance on sub-sets of the LAMA probe for large values of k by adding as little as 2.1% additional parameters to the original models.</abstract>
<identifier type="citekey">wold-2022-effectiveness</identifier>
<location>
<url>https://aclanthology.org/2022.textgraphs-1.6</url>
</location>
<part>
<date>2022-10</date>
<extent unit="page">
<start>54</start>
<end>59</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Effectiveness of Masked Language Modeling and Adapters for Factual Knowledge Injection
%A Wold, Sondre
%Y Ustalov, Dmitry
%Y Gao, Yanjun
%Y Panchenko, Alexander
%Y Valentino, Marco
%Y Thayaparan, Mokanarangan
%Y Nguyen, Thien Huu
%Y Penn, Gerald
%Y Ramesh, Arti
%Y Jana, Abhik
%S Proceedings of TextGraphs-16: Graph-based Methods for Natural Language Processing
%D 2022
%8 October
%I Association for Computational Linguistics
%C Gyeongju, Republic of Korea
%F wold-2022-effectiveness
%X This paper studies the problem of injecting factual knowledge into large pre-trained language models. We train adapter modules on parts of the ConceptNet knowledge graph using the masked language modeling objective and evaluate the success of the method by a series of probing experiments on the LAMA probe. Mean P@K curves for different configurations indicate that the technique is effective, increasing the performance on sub-sets of the LAMA probe for large values of k by adding as little as 2.1% additional parameters to the original models.
%U https://aclanthology.org/2022.textgraphs-1.6
%P 54-59
Markdown (Informal)
[The Effectiveness of Masked Language Modeling and Adapters for Factual Knowledge Injection](https://aclanthology.org/2022.textgraphs-1.6) (Wold, TextGraphs 2022)
ACL