@inproceedings{schott-etal-2023-polyglot,
title = "Polyglot or Not? Measuring Multilingual Encyclopedic Knowledge in Foundation Models",
author = "Schott, Tim and
Furman, Daniel and
Bhat, Shreshta",
editor = "Bouamor, Houda and
Pino, Juan and
Bali, Kalika",
booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing",
month = dec,
year = "2023",
address = "Singapore",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2023.emnlp-main.691",
doi = "10.18653/v1/2023.emnlp-main.691",
pages = "11238--11253",
abstract = "In this work, we assess the ability of foundation models to recall encyclopedic knowledge across a wide range of linguistic contexts. To support this, we: 1) produce a 20-language dataset that contains 303k factual associations paired with counterfactuals, 2) evaluate 5 models in a multilingual test, and 3) benchmark a diverse set of 24 models in an English-only test. Meta{'}s LLaMA achieves the highest scores in both multilingual and English-only evaluations. Yet, an analysis of LLaMA{'}s errors reveals significant limitations in its ability to recall facts in languages other than English, plus difficulties related to the location and gender of fact subjects. Overall, our findings suggest that today{'}s foundation models are far from polyglots.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="schott-etal-2023-polyglot">
<titleInfo>
<title>Polyglot or Not? Measuring Multilingual Encyclopedic Knowledge in Foundation Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Tim</namePart>
<namePart type="family">Schott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Furman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shreshta</namePart>
<namePart type="family">Bhat</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2023-12</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Houda</namePart>
<namePart type="family">Bouamor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Juan</namePart>
<namePart type="family">Pino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kalika</namePart>
<namePart type="family">Bali</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Singapore</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>In this work, we assess the ability of foundation models to recall encyclopedic knowledge across a wide range of linguistic contexts. To support this, we: 1) produce a 20-language dataset that contains 303k factual associations paired with counterfactuals, 2) evaluate 5 models in a multilingual test, and 3) benchmark a diverse set of 24 models in an English-only test. Meta’s LLaMA achieves the highest scores in both multilingual and English-only evaluations. Yet, an analysis of LLaMA’s errors reveals significant limitations in its ability to recall facts in languages other than English, plus difficulties related to the location and gender of fact subjects. Overall, our findings suggest that today’s foundation models are far from polyglots.</abstract>
<identifier type="citekey">schott-etal-2023-polyglot</identifier>
<identifier type="doi">10.18653/v1/2023.emnlp-main.691</identifier>
<location>
<url>https://aclanthology.org/2023.emnlp-main.691</url>
</location>
<part>
<date>2023-12</date>
<extent unit="page">
<start>11238</start>
<end>11253</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Polyglot or Not? Measuring Multilingual Encyclopedic Knowledge in Foundation Models
%A Schott, Tim
%A Furman, Daniel
%A Bhat, Shreshta
%Y Bouamor, Houda
%Y Pino, Juan
%Y Bali, Kalika
%S Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing
%D 2023
%8 December
%I Association for Computational Linguistics
%C Singapore
%F schott-etal-2023-polyglot
%X In this work, we assess the ability of foundation models to recall encyclopedic knowledge across a wide range of linguistic contexts. To support this, we: 1) produce a 20-language dataset that contains 303k factual associations paired with counterfactuals, 2) evaluate 5 models in a multilingual test, and 3) benchmark a diverse set of 24 models in an English-only test. Meta’s LLaMA achieves the highest scores in both multilingual and English-only evaluations. Yet, an analysis of LLaMA’s errors reveals significant limitations in its ability to recall facts in languages other than English, plus difficulties related to the location and gender of fact subjects. Overall, our findings suggest that today’s foundation models are far from polyglots.
%R 10.18653/v1/2023.emnlp-main.691
%U https://aclanthology.org/2023.emnlp-main.691
%U https://doi.org/10.18653/v1/2023.emnlp-main.691
%P 11238-11253
Markdown (Informal)
[Polyglot or Not? Measuring Multilingual Encyclopedic Knowledge in Foundation Models](https://aclanthology.org/2023.emnlp-main.691) (Schott et al., EMNLP 2023)
ACL