@inproceedings{geromel-cimiano-2026-improving,
title = "Improving {LLM}-Based Entity Detection with Annotation Guidelines",
author = "Geromel, Marcel and
Cimiano, Philipp",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.234/",
pages = "5251--5268",
ISBN = "979-8-89176-439-2",
abstract = "In this work, we investigate whether detailed, dataset-specific annotation guidelines can replace or augment annotated few-shot exemplars for prompt-based entity detection. To this end, we propose an in-context learning framework that conditions the LLM on a guideline document, task instructions, and optional demonstrations, enabling single-pass, primarily zero-shot nested entity detection. We evaluate multiple instructional aspects from the comprehensive ACE 2005 annotation guidelines, using three guideline-compliant datasets as benchmarks and GPT-5 variants for inference. Based on extensive experimentation, we demonstrate that detailed, dataset-specific instructions, especially those governing mention boundaries, consistently and robustly improve performance, even in conjunction with established few-shot methods. Indeed, even under zero-shot constraints, our guideline approach outperforms state-of-the-art few-shot methods on ACE-2005, achieving up to 81.2 percent F1 without relying on existing labeled datasets. Our findings present detailed annotation guidelines as a promising resource for improving instruction-based information extraction."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="geromel-cimiano-2026-improving">
<titleInfo>
<title>Improving LLM-Based Entity Detection with Annotation Guidelines</title>
</titleInfo>
<name type="personal">
<namePart type="given">Marcel</namePart>
<namePart type="family">Geromel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Philipp</namePart>
<namePart type="family">Cimiano</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>In this work, we investigate whether detailed, dataset-specific annotation guidelines can replace or augment annotated few-shot exemplars for prompt-based entity detection. To this end, we propose an in-context learning framework that conditions the LLM on a guideline document, task instructions, and optional demonstrations, enabling single-pass, primarily zero-shot nested entity detection. We evaluate multiple instructional aspects from the comprehensive ACE 2005 annotation guidelines, using three guideline-compliant datasets as benchmarks and GPT-5 variants for inference. Based on extensive experimentation, we demonstrate that detailed, dataset-specific instructions, especially those governing mention boundaries, consistently and robustly improve performance, even in conjunction with established few-shot methods. Indeed, even under zero-shot constraints, our guideline approach outperforms state-of-the-art few-shot methods on ACE-2005, achieving up to 81.2 percent F1 without relying on existing labeled datasets. Our findings present detailed annotation guidelines as a promising resource for improving instruction-based information extraction.</abstract>
<identifier type="citekey">geromel-cimiano-2026-improving</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.234/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>5251</start>
<end>5268</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Improving LLM-Based Entity Detection with Annotation Guidelines
%A Geromel, Marcel
%A Cimiano, Philipp
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F geromel-cimiano-2026-improving
%X In this work, we investigate whether detailed, dataset-specific annotation guidelines can replace or augment annotated few-shot exemplars for prompt-based entity detection. To this end, we propose an in-context learning framework that conditions the LLM on a guideline document, task instructions, and optional demonstrations, enabling single-pass, primarily zero-shot nested entity detection. We evaluate multiple instructional aspects from the comprehensive ACE 2005 annotation guidelines, using three guideline-compliant datasets as benchmarks and GPT-5 variants for inference. Based on extensive experimentation, we demonstrate that detailed, dataset-specific instructions, especially those governing mention boundaries, consistently and robustly improve performance, even in conjunction with established few-shot methods. Indeed, even under zero-shot constraints, our guideline approach outperforms state-of-the-art few-shot methods on ACE-2005, achieving up to 81.2 percent F1 without relying on existing labeled datasets. Our findings present detailed annotation guidelines as a promising resource for improving instruction-based information extraction.
%U https://aclanthology.org/2026.aacl-long.234/
%P 5251-5268
Markdown (Informal)
[Improving LLM-Based Entity Detection with Annotation Guidelines](https://aclanthology.org/2026.aacl-long.234/) (Geromel & Cimiano, AACL-IJCNLP 2026)
ACL
- Marcel Geromel and Philipp Cimiano. 2026. Improving LLM-Based Entity Detection with Annotation Guidelines. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 5251–5268, Hengqin, China. Association for Computational Linguistics.