@inproceedings{pappacoda-etal-2026-generscore,
title = "{G}e{NER}score: Semantic Alignment Metrics for Generative Named Entity Recognition",
author = "Pappacoda, Gianmarco and
Ruggeri, Federico and
Torroni, Paolo",
editor = "Bak, JinYeong and
van der Goot, Rob and
Jang, Hyeju and
Buaphet, Weerayut and
Ramponi, Alan and
Xu, Wei and
Ritter, Alan",
booktitle = "Proceedings of the 11th Workshop on Natural User-generated Text",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.wnut-1.9/",
pages = "117--140",
ISBN = "979-8-89176-470-5",
abstract = "Traditional Named Entity Recognition (NER) metrics rely on syntactic span and type matching. While this may be appropriate if NER is framed as a sequence labeling task, the advent of generative models allows for alternative approaches, where model outputs are not extracted, but generated. In this perspective, generative NER calls for more semantically-aware scoring mechanisms. Generative models may produce valid aliases, paraphrases, or type variants that differ from the annotated gold, while also producing unsupported predictions. We propose a taxonomy of such cases and introduce an embedding-based parametric method to evaluate generative NER based on three metrics: soft precision, soft recall and unsupported rate. We validate this framework using standard and noisy datasets, across state-of-the-art systems such as GPT and GLiNER alongside older encoder-based models. We find that our metrics mostly agree with traditional ones on better-established, less noisy benchmarks, but differ in more recent, noisier ones, and can capture edge cases that arise from free-form generation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="pappacoda-etal-2026-generscore">
<titleInfo>
<title>GeNERscore: Semantic Alignment Metrics for Generative Named Entity Recognition</title>
</titleInfo>
<name type="personal">
<namePart type="given">Gianmarco</namePart>
<namePart type="family">Pappacoda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Federico</namePart>
<namePart type="family">Ruggeri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Paolo</namePart>
<namePart type="family">Torroni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 11th Workshop on Natural User-generated Text</title>
</titleInfo>
<name type="personal">
<namePart type="given">JinYeong</namePart>
<namePart type="family">Bak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rob</namePart>
<namePart type="family">van der Goot</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hyeju</namePart>
<namePart type="family">Jang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weerayut</namePart>
<namePart type="family">Buaphet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Ramponi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Ritter</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-470-5</identifier>
</relatedItem>
<abstract>Traditional Named Entity Recognition (NER) metrics rely on syntactic span and type matching. While this may be appropriate if NER is framed as a sequence labeling task, the advent of generative models allows for alternative approaches, where model outputs are not extracted, but generated. In this perspective, generative NER calls for more semantically-aware scoring mechanisms. Generative models may produce valid aliases, paraphrases, or type variants that differ from the annotated gold, while also producing unsupported predictions. We propose a taxonomy of such cases and introduce an embedding-based parametric method to evaluate generative NER based on three metrics: soft precision, soft recall and unsupported rate. We validate this framework using standard and noisy datasets, across state-of-the-art systems such as GPT and GLiNER alongside older encoder-based models. We find that our metrics mostly agree with traditional ones on better-established, less noisy benchmarks, but differ in more recent, noisier ones, and can capture edge cases that arise from free-form generation.</abstract>
<identifier type="citekey">pappacoda-etal-2026-generscore</identifier>
<location>
<url>https://aclanthology.org/2026.wnut-1.9/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>117</start>
<end>140</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T GeNERscore: Semantic Alignment Metrics for Generative Named Entity Recognition
%A Pappacoda, Gianmarco
%A Ruggeri, Federico
%A Torroni, Paolo
%Y Bak, JinYeong
%Y van der Goot, Rob
%Y Jang, Hyeju
%Y Buaphet, Weerayut
%Y Ramponi, Alan
%Y Xu, Wei
%Y Ritter, Alan
%S Proceedings of the 11th Workshop on Natural User-generated Text
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-470-5
%F pappacoda-etal-2026-generscore
%X Traditional Named Entity Recognition (NER) metrics rely on syntactic span and type matching. While this may be appropriate if NER is framed as a sequence labeling task, the advent of generative models allows for alternative approaches, where model outputs are not extracted, but generated. In this perspective, generative NER calls for more semantically-aware scoring mechanisms. Generative models may produce valid aliases, paraphrases, or type variants that differ from the annotated gold, while also producing unsupported predictions. We propose a taxonomy of such cases and introduce an embedding-based parametric method to evaluate generative NER based on three metrics: soft precision, soft recall and unsupported rate. We validate this framework using standard and noisy datasets, across state-of-the-art systems such as GPT and GLiNER alongside older encoder-based models. We find that our metrics mostly agree with traditional ones on better-established, less noisy benchmarks, but differ in more recent, noisier ones, and can capture edge cases that arise from free-form generation.
%U https://aclanthology.org/2026.wnut-1.9/
%P 117-140
Markdown (Informal)
[GeNERscore: Semantic Alignment Metrics for Generative Named Entity Recognition](https://aclanthology.org/2026.wnut-1.9/) (Pappacoda et al., WNUT 2026)
ACL