@inproceedings{purason-etal-2026-smugri,
title = "{SMUGRI}-{COPA}: Evaluating Commonsense Reasoning for Low-Resource Finnic Languages",
author = "Purason, Taido and
Tomingas, Marili and
Vaab, Janek and
Fishel, Mark",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2080/",
pages = "44289--44303",
ISBN = "979-8-89176-462-0",
abstract = "We introduce SMUGRI-COPA, a commonsense reasoning benchmark for two heavily under-resourced Finnic languages: V{\~o}ro and Livonian. Existing evaluation resources for these languages rely on a translated subset of FLORES sentences, limiting linguistic diversity and the evaluation of broader language understanding. To address this gap, we human-translate the COPA (Choice of Plausible Alternatives) benchmark into V{\~o}ro and Livonian, creating, to our knowledge, the largest discriminative benchmark currently available for these languages. We evaluate a wide range of proprietary and open-weight language models on SMUGRI-COPA and compare the results with Estonian XCOPA. Our results reveal a substantial performance gap between Estonian and the lower-resource languages, with most models failing to significantly outperform random guessing on Livonian. We further find that enabling reasoning for GPT-5 substantially improves accuracy for V{\~o}ro and Livonian, while its reasoning summaries show frequent reliance on explicit translation of the tasks. Our work highlights the challenges of evaluating endangered language understanding and provides a new resource for multilingual evaluation research."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="purason-etal-2026-smugri">
<titleInfo>
<title>SMUGRI-COPA: Evaluating Commonsense Reasoning for Low-Resource Finnic Languages</title>
</titleInfo>
<name type="personal">
<namePart type="given">Taido</namePart>
<namePart type="family">Purason</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marili</namePart>
<namePart type="family">Tomingas</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Janek</namePart>
<namePart type="family">Vaab</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mark</namePart>
<namePart type="family">Fishel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>We introduce SMUGRI-COPA, a commonsense reasoning benchmark for two heavily under-resourced Finnic languages: Võro and Livonian. Existing evaluation resources for these languages rely on a translated subset of FLORES sentences, limiting linguistic diversity and the evaluation of broader language understanding. To address this gap, we human-translate the COPA (Choice of Plausible Alternatives) benchmark into Võro and Livonian, creating, to our knowledge, the largest discriminative benchmark currently available for these languages. We evaluate a wide range of proprietary and open-weight language models on SMUGRI-COPA and compare the results with Estonian XCOPA. Our results reveal a substantial performance gap between Estonian and the lower-resource languages, with most models failing to significantly outperform random guessing on Livonian. We further find that enabling reasoning for GPT-5 substantially improves accuracy for Võro and Livonian, while its reasoning summaries show frequent reliance on explicit translation of the tasks. Our work highlights the challenges of evaluating endangered language understanding and provides a new resource for multilingual evaluation research.</abstract>
<identifier type="citekey">purason-etal-2026-smugri</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2080/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>44289</start>
<end>44303</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SMUGRI-COPA: Evaluating Commonsense Reasoning for Low-Resource Finnic Languages
%A Purason, Taido
%A Tomingas, Marili
%A Vaab, Janek
%A Fishel, Mark
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F purason-etal-2026-smugri
%X We introduce SMUGRI-COPA, a commonsense reasoning benchmark for two heavily under-resourced Finnic languages: Võro and Livonian. Existing evaluation resources for these languages rely on a translated subset of FLORES sentences, limiting linguistic diversity and the evaluation of broader language understanding. To address this gap, we human-translate the COPA (Choice of Plausible Alternatives) benchmark into Võro and Livonian, creating, to our knowledge, the largest discriminative benchmark currently available for these languages. We evaluate a wide range of proprietary and open-weight language models on SMUGRI-COPA and compare the results with Estonian XCOPA. Our results reveal a substantial performance gap between Estonian and the lower-resource languages, with most models failing to significantly outperform random guessing on Livonian. We further find that enabling reasoning for GPT-5 substantially improves accuracy for Võro and Livonian, while its reasoning summaries show frequent reliance on explicit translation of the tasks. Our work highlights the challenges of evaluating endangered language understanding and provides a new resource for multilingual evaluation research.
%U https://aclanthology.org/2026.findings-emnlp.2080/
%P 44289-44303
Markdown (Informal)
[SMUGRI-COPA: Evaluating Commonsense Reasoning for Low-Resource Finnic Languages](https://aclanthology.org/2026.findings-emnlp.2080/) (Purason et al., Findings 2026)
ACL