@inproceedings{belcavello-etal-2026-evaluating,
title = "Evaluating the Impact of {LLM}-Assisted Annotation in a Perspectivized Setting: The Case of {F}rame{N}et Annotation",
author = "Belcavello, Frederico and
Matos, Ely E. and
Lorenzi, Arthur and
Bonoto, Lisandra and
P{\'a}dua Ruiz, Livia and
Pereira, Luiz Fernando and
Herbst, Victor and
Navarro, Yulla Liquer and
Abreu, Helen de Andrade and
Vicente Dutra, L{\'i}via and
Torrent, Tiago Timponi",
editor = "Bunt, Harry",
booktitle = "Proceedings of the 22nd Joint {ACL} - {ISO} Workshop on Interoperable Semantic Annotation and Representation ({ISA}-22) @ {LREC} 2026",
month = may,
year = "2026",
address = "Palma, Mallorca (Spain)",
publisher = "ELRA Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.isa-1.9/",
doi = "10.63317/36ot28es26u8",
pages = "77--87",
abstract = "The use of LLM-based applications as a means to accelerate and/or substitute human labor in the creation of language resources and datasets is a reality. Nonetheless, despite the potential of such tools for linguistic research, an evaluation of their performance and impact on the creation of annotated datasets, especially under a perspectivized approach to NLP, is still missing. This paper contributes to the reduction of this gap by reporting on an extensive evaluation of the (semi-)automatization of FrameNet-like semantic annotation by the use of an LLM-based semantic role labeler. The methodology employed compares annotation time, coverage, and diversity in three experimental settings: manual, automatic, and semi-automatic annotation. Results show that the hybrid, semi-automatic annotation setting leads to increased frame diversity and similar annotation coverage, when compared to the human-only setting, while the automatic setting performs considerably worse in all metrics, except for annotation time, which remains similar."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="belcavello-etal-2026-evaluating">
<titleInfo>
<title>Evaluating the Impact of LLM-Assisted Annotation in a Perspectivized Setting: The Case of FrameNet Annotation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Frederico</namePart>
<namePart type="family">Belcavello</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ely</namePart>
<namePart type="given">E</namePart>
<namePart type="family">Matos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Arthur</namePart>
<namePart type="family">Lorenzi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lisandra</namePart>
<namePart type="family">Bonoto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Livia</namePart>
<namePart type="family">Pádua Ruiz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Luiz</namePart>
<namePart type="given">Fernando</namePart>
<namePart type="family">Pereira</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Victor</namePart>
<namePart type="family">Herbst</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yulla</namePart>
<namePart type="given">Liquer</namePart>
<namePart type="family">Navarro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Helen</namePart>
<namePart type="given">de</namePart>
<namePart type="given">Andrade</namePart>
<namePart type="family">Abreu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lívia</namePart>
<namePart type="family">Vicente Dutra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tiago</namePart>
<namePart type="given">Timponi</namePart>
<namePart type="family">Torrent</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 22nd Joint ACL - ISO Workshop on Interoperable Semantic Annotation and Representation (ISA-22) @ LREC 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Harry</namePart>
<namePart type="family">Bunt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma, Mallorca (Spain)</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>The use of LLM-based applications as a means to accelerate and/or substitute human labor in the creation of language resources and datasets is a reality. Nonetheless, despite the potential of such tools for linguistic research, an evaluation of their performance and impact on the creation of annotated datasets, especially under a perspectivized approach to NLP, is still missing. This paper contributes to the reduction of this gap by reporting on an extensive evaluation of the (semi-)automatization of FrameNet-like semantic annotation by the use of an LLM-based semantic role labeler. The methodology employed compares annotation time, coverage, and diversity in three experimental settings: manual, automatic, and semi-automatic annotation. Results show that the hybrid, semi-automatic annotation setting leads to increased frame diversity and similar annotation coverage, when compared to the human-only setting, while the automatic setting performs considerably worse in all metrics, except for annotation time, which remains similar.</abstract>
<identifier type="citekey">belcavello-etal-2026-evaluating</identifier>
<identifier type="doi">10.63317/36ot28es26u8</identifier>
<location>
<url>https://aclanthology.org/2026.isa-1.9/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>77</start>
<end>87</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating the Impact of LLM-Assisted Annotation in a Perspectivized Setting: The Case of FrameNet Annotation
%A Belcavello, Frederico
%A Matos, Ely E.
%A Lorenzi, Arthur
%A Bonoto, Lisandra
%A Pádua Ruiz, Livia
%A Pereira, Luiz Fernando
%A Herbst, Victor
%A Navarro, Yulla Liquer
%A Abreu, Helen de Andrade
%A Vicente Dutra, Lívia
%A Torrent, Tiago Timponi
%Y Bunt, Harry
%S Proceedings of the 22nd Joint ACL - ISO Workshop on Interoperable Semantic Annotation and Representation (ISA-22) @ LREC 2026
%D 2026
%8 May
%I ELRA Language Resources Association (ELRA)
%C Palma, Mallorca (Spain)
%F belcavello-etal-2026-evaluating
%X The use of LLM-based applications as a means to accelerate and/or substitute human labor in the creation of language resources and datasets is a reality. Nonetheless, despite the potential of such tools for linguistic research, an evaluation of their performance and impact on the creation of annotated datasets, especially under a perspectivized approach to NLP, is still missing. This paper contributes to the reduction of this gap by reporting on an extensive evaluation of the (semi-)automatization of FrameNet-like semantic annotation by the use of an LLM-based semantic role labeler. The methodology employed compares annotation time, coverage, and diversity in three experimental settings: manual, automatic, and semi-automatic annotation. Results show that the hybrid, semi-automatic annotation setting leads to increased frame diversity and similar annotation coverage, when compared to the human-only setting, while the automatic setting performs considerably worse in all metrics, except for annotation time, which remains similar.
%R 10.63317/36ot28es26u8
%U https://aclanthology.org/2026.isa-1.9/
%U https://doi.org/10.63317/36ot28es26u8
%P 77-87
Markdown (Informal)
[Evaluating the Impact of LLM-Assisted Annotation in a Perspectivized Setting: The Case of FrameNet Annotation](https://aclanthology.org/2026.isa-1.9/) (Belcavello et al., ISA 2026)
ACL
- Frederico Belcavello, Ely E. Matos, Arthur Lorenzi, Lisandra Bonoto, Livia Pádua Ruiz, Luiz Fernando Pereira, Victor Herbst, Yulla Liquer Navarro, Helen de Andrade Abreu, Lívia Vicente Dutra, and Tiago Timponi Torrent. 2026. Evaluating the Impact of LLM-Assisted Annotation in a Perspectivized Setting: The Case of FrameNet Annotation. In Proceedings of the 22nd Joint ACL - ISO Workshop on Interoperable Semantic Annotation and Representation (ISA-22) @ LREC 2026, pages 77–87, Palma, Mallorca (Spain). ELRA Language Resources Association (ELRA).