@inproceedings{debess-etal-2026-words,
title = "Of Words and Meaning: A Grammatical and Semantic Benchmark for {F}aroese {LLM} Understanding",
author = "Debess, Iben Nyholm and
Scalvini, Barbara and
Pedersen, Bolette",
editor = "Piperidis, Stelios and
Bel, N{\'u}ria and
van den Heuvel, Henk and
Ide, Nancy and
Krek, Simon and
Toral, Antonio",
booktitle = "Proceedings of the Fifteenth Language Resources and Evaluation Conference",
month = may,
year = "2026",
address = "Palma de Mallorca, Spain",
publisher = "ELRA Language Resource Association",
url = "https://aclanthology.org/2026.lrec-1.354/",
doi = "10.63317/4u4i99hc8co8",
pages = "4514--4526",
abstract = "Evaluating language technology for low-resource languages faces a fundamental challenge: the scarcity of native benchmarks suitable for systematic assessment. For Faroese, no such evaluation frameworks exist. We address this gap by presenting the first benchmark suite for Faroese semantic understanding and grammatical competence. Our methodology transforms existing lexicographic resources, authoritative dictionaries and error corpora, into systematic evaluation tasks through computational restructuring, demonstrating a replicable approach for resource-constrained settings. The resulting benchmarks assess grammatical correctness, semantic relation classification, and metaphor comprehension. Evaluation across LLMs from compact open-source to large-scale commercial systems reveals consistent performance patterns favouring proprietary models. This work establishes a proof of concept for benchmark creation from traditional linguistic resources, and provides a methodological template for other low-resource language communities."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="debess-etal-2026-words">
<titleInfo>
<title>Of Words and Meaning: A Grammatical and Semantic Benchmark for Faroese LLM Understanding</title>
</titleInfo>
<name type="personal">
<namePart type="given">Iben</namePart>
<namePart type="given">Nyholm</namePart>
<namePart type="family">Debess</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Scalvini</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bolette</namePart>
<namePart type="family">Pedersen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Fifteenth Language Resources and Evaluation Conference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Stelios</namePart>
<namePart type="family">Piperidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Núria</namePart>
<namePart type="family">Bel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Henk</namePart>
<namePart type="family">van den Heuvel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nancy</namePart>
<namePart type="family">Ide</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Krek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Antonio</namePart>
<namePart type="family">Toral</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resource Association</publisher>
<place>
<placeTerm type="text">Palma de Mallorca, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Evaluating language technology for low-resource languages faces a fundamental challenge: the scarcity of native benchmarks suitable for systematic assessment. For Faroese, no such evaluation frameworks exist. We address this gap by presenting the first benchmark suite for Faroese semantic understanding and grammatical competence. Our methodology transforms existing lexicographic resources, authoritative dictionaries and error corpora, into systematic evaluation tasks through computational restructuring, demonstrating a replicable approach for resource-constrained settings. The resulting benchmarks assess grammatical correctness, semantic relation classification, and metaphor comprehension. Evaluation across LLMs from compact open-source to large-scale commercial systems reveals consistent performance patterns favouring proprietary models. This work establishes a proof of concept for benchmark creation from traditional linguistic resources, and provides a methodological template for other low-resource language communities.</abstract>
<identifier type="citekey">debess-etal-2026-words</identifier>
<identifier type="doi">10.63317/4u4i99hc8co8</identifier>
<location>
<url>https://aclanthology.org/2026.lrec-1.354/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>4514</start>
<end>4526</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Of Words and Meaning: A Grammatical and Semantic Benchmark for Faroese LLM Understanding
%A Debess, Iben Nyholm
%A Scalvini, Barbara
%A Pedersen, Bolette
%Y Piperidis, Stelios
%Y Bel, Núria
%Y van den Heuvel, Henk
%Y Ide, Nancy
%Y Krek, Simon
%Y Toral, Antonio
%S Proceedings of the Fifteenth Language Resources and Evaluation Conference
%D 2026
%8 May
%I ELRA Language Resource Association
%C Palma de Mallorca, Spain
%F debess-etal-2026-words
%X Evaluating language technology for low-resource languages faces a fundamental challenge: the scarcity of native benchmarks suitable for systematic assessment. For Faroese, no such evaluation frameworks exist. We address this gap by presenting the first benchmark suite for Faroese semantic understanding and grammatical competence. Our methodology transforms existing lexicographic resources, authoritative dictionaries and error corpora, into systematic evaluation tasks through computational restructuring, demonstrating a replicable approach for resource-constrained settings. The resulting benchmarks assess grammatical correctness, semantic relation classification, and metaphor comprehension. Evaluation across LLMs from compact open-source to large-scale commercial systems reveals consistent performance patterns favouring proprietary models. This work establishes a proof of concept for benchmark creation from traditional linguistic resources, and provides a methodological template for other low-resource language communities.
%R 10.63317/4u4i99hc8co8
%U https://aclanthology.org/2026.lrec-1.354/
%U https://doi.org/10.63317/4u4i99hc8co8
%P 4514-4526
Markdown (Informal)
[Of Words and Meaning: A Grammatical and Semantic Benchmark for Faroese LLM Understanding](https://aclanthology.org/2026.lrec-1.354/) (Debess et al., LREC 2026)
ACL