@inproceedings{arnardottir-etal-2024-beyond,
title = "Beyond Error Categories: A Contextual Approach of Evaluating Emerging Spell and Grammar Checkers",
author = "Arnard{\'o}ttir, {\TH}{\'o}runn and
Ing{\'o}lfsd{\'o}ttir, Svanhv{\'\i}t Lilja and
S{\'\i}monarson, Haukur Barri and
Einarsson, Hafsteinn and
Ingason, Anton Karl and
{\TH}orsteinsson, Vilhj{\'a}lmur",
editor = "Melero, Maite and
Sakti, Sakriani and
Soria, Claudia",
booktitle = "Proceedings of the 3rd Annual Meeting of the Special Interest Group on Under-resourced Languages @ LREC-COLING 2024",
month = may,
year = "2024",
address = "Torino, Italia",
publisher = "ELRA and ICCL",
url = "https://aclanthology.org/2024.sigul-1.6",
pages = "45--52",
abstract = "Automatic spell and grammar checking can be done using various system architectures, and large language models have recently been used to solve the task with promising results. Here we describe a new method of creating test data to measure the performance of spell and grammar checkers, including large language models. Three types of test data represent different approaches to evaluation, from basic error detection to error correction with natural language explanations of the corrections made and error severity scores, which is the main novelty of this approach. These additions are especially useful when evaluating large language models. We present a spell and grammar checking test set for Icelandic in which the described approach is applied. The data consists of whole texts instead of discrete sentences, which facilitates evaluating context awareness of models. The resulting test set can be used to compare different spell and grammar checkers and is published under permissive licenses.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="arnardottir-etal-2024-beyond">
<titleInfo>
<title>Beyond Error Categories: A Contextual Approach of Evaluating Emerging Spell and Grammar Checkers</title>
</titleInfo>
<name type="personal">
<namePart type="given">\THórunn</namePart>
<namePart type="family">Arnardóttir</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Svanhvít</namePart>
<namePart type="given">Lilja</namePart>
<namePart type="family">Ingólfsdóttir</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haukur</namePart>
<namePart type="given">Barri</namePart>
<namePart type="family">Símonarson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hafsteinn</namePart>
<namePart type="family">Einarsson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anton</namePart>
<namePart type="given">Karl</namePart>
<namePart type="family">Ingason</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vilhjálmur</namePart>
<namePart type="family">\THorsteinsson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2024-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 3rd Annual Meeting of the Special Interest Group on Under-resourced Languages @ LREC-COLING 2024</title>
</titleInfo>
<name type="personal">
<namePart type="given">Maite</namePart>
<namePart type="family">Melero</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sakriani</namePart>
<namePart type="family">Sakti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Claudia</namePart>
<namePart type="family">Soria</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA and ICCL</publisher>
<place>
<placeTerm type="text">Torino, Italia</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Automatic spell and grammar checking can be done using various system architectures, and large language models have recently been used to solve the task with promising results. Here we describe a new method of creating test data to measure the performance of spell and grammar checkers, including large language models. Three types of test data represent different approaches to evaluation, from basic error detection to error correction with natural language explanations of the corrections made and error severity scores, which is the main novelty of this approach. These additions are especially useful when evaluating large language models. We present a spell and grammar checking test set for Icelandic in which the described approach is applied. The data consists of whole texts instead of discrete sentences, which facilitates evaluating context awareness of models. The resulting test set can be used to compare different spell and grammar checkers and is published under permissive licenses.</abstract>
<identifier type="citekey">arnardottir-etal-2024-beyond</identifier>
<location>
<url>https://aclanthology.org/2024.sigul-1.6</url>
</location>
<part>
<date>2024-05</date>
<extent unit="page">
<start>45</start>
<end>52</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Error Categories: A Contextual Approach of Evaluating Emerging Spell and Grammar Checkers
%A Arnardóttir, \THórunn
%A Ingólfsdóttir, Svanhvít Lilja
%A Símonarson, Haukur Barri
%A Einarsson, Hafsteinn
%A Ingason, Anton Karl
%A \THorsteinsson, Vilhjálmur
%Y Melero, Maite
%Y Sakti, Sakriani
%Y Soria, Claudia
%S Proceedings of the 3rd Annual Meeting of the Special Interest Group on Under-resourced Languages @ LREC-COLING 2024
%D 2024
%8 May
%I ELRA and ICCL
%C Torino, Italia
%F arnardottir-etal-2024-beyond
%X Automatic spell and grammar checking can be done using various system architectures, and large language models have recently been used to solve the task with promising results. Here we describe a new method of creating test data to measure the performance of spell and grammar checkers, including large language models. Three types of test data represent different approaches to evaluation, from basic error detection to error correction with natural language explanations of the corrections made and error severity scores, which is the main novelty of this approach. These additions are especially useful when evaluating large language models. We present a spell and grammar checking test set for Icelandic in which the described approach is applied. The data consists of whole texts instead of discrete sentences, which facilitates evaluating context awareness of models. The resulting test set can be used to compare different spell and grammar checkers and is published under permissive licenses.
%U https://aclanthology.org/2024.sigul-1.6
%P 45-52
Markdown (Informal)
[Beyond Error Categories: A Contextual Approach of Evaluating Emerging Spell and Grammar Checkers](https://aclanthology.org/2024.sigul-1.6) (Arnardóttir et al., SIGUL-WS 2024)
ACL