@inproceedings{kolk-etal-2026-evaluating,
title = "Evaluating Large Language Models for Strategic Knowledge Extraction in Capability-Based Planning",
author = "Kolk, Hein C. and
Garc{\'i}a-Fern{\'a}ndez, Julia and
Bronkhorst, Julia and
Bakker, Roos M.",
editor = "S{\'e}rasset, Gilles and
Gkirtzou, Katerina and
Cochez, Michael and
Kalo, Jan-Christoph",
booktitle = "Proceedings of the Knowledge Graphs and Large Language Models Workshop ({KG}-{LLM}) @ {LREC}26",
month = may,
year = "2026",
address = "Palma, Mallorca (Spain)",
publisher = "ELRA Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.kallm-1.20/",
doi = "10.63317/4pp744dg8ixs",
pages = "210--220",
abstract = "In a security environment that is growing more complex, large national organizations like the police rely on strategic frameworks to guide their decision-making. Frameworks like the Capability Based Planning (CBP) system are used to address this, but require a vast amount of information to function properly. A significant but underused store of information lies within an organization{'}s own internal flow of documents, like vision statements or annual reports. We tap into this flow by proposing a method to automatically extract relevant strategic entities and structuring them within a knowledge graph. We evaluate the performance of various Large Language Models (LLMs) on a corpus of policy excerpts from the Dutch National Police in extracting relevant strategic entities and linking them to core police capabilities. We employ the novel alternative annotator test (Alt-Test) to determine if an LLM can serve as a reliable substitute for a human domain expert on this highly subjective task. Our evaluation shows that while LLMs cannot fully replace human experts, they prove to be valuable support tools by frequently identifying the same strategic information as the annotators, successfully extracting core entities and linking them to predefined capabilities."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kolk-etal-2026-evaluating">
<titleInfo>
<title>Evaluating Large Language Models for Strategic Knowledge Extraction in Capability-Based Planning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hein</namePart>
<namePart type="given">C</namePart>
<namePart type="family">Kolk</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Julia</namePart>
<namePart type="family">García-Fernández</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Julia</namePart>
<namePart type="family">Bronkhorst</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Roos</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Bakker</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Knowledge Graphs and Large Language Models Workshop (KG-LLM) @ LREC26</title>
</titleInfo>
<name type="personal">
<namePart type="given">Gilles</namePart>
<namePart type="family">Sérasset</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Katerina</namePart>
<namePart type="family">Gkirtzou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Michael</namePart>
<namePart type="family">Cochez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jan-Christoph</namePart>
<namePart type="family">Kalo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma, Mallorca (Spain)</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>In a security environment that is growing more complex, large national organizations like the police rely on strategic frameworks to guide their decision-making. Frameworks like the Capability Based Planning (CBP) system are used to address this, but require a vast amount of information to function properly. A significant but underused store of information lies within an organization’s own internal flow of documents, like vision statements or annual reports. We tap into this flow by proposing a method to automatically extract relevant strategic entities and structuring them within a knowledge graph. We evaluate the performance of various Large Language Models (LLMs) on a corpus of policy excerpts from the Dutch National Police in extracting relevant strategic entities and linking them to core police capabilities. We employ the novel alternative annotator test (Alt-Test) to determine if an LLM can serve as a reliable substitute for a human domain expert on this highly subjective task. Our evaluation shows that while LLMs cannot fully replace human experts, they prove to be valuable support tools by frequently identifying the same strategic information as the annotators, successfully extracting core entities and linking them to predefined capabilities.</abstract>
<identifier type="citekey">kolk-etal-2026-evaluating</identifier>
<identifier type="doi">10.63317/4pp744dg8ixs</identifier>
<location>
<url>https://aclanthology.org/2026.kallm-1.20/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>210</start>
<end>220</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating Large Language Models for Strategic Knowledge Extraction in Capability-Based Planning
%A Kolk, Hein C.
%A García-Fernández, Julia
%A Bronkhorst, Julia
%A Bakker, Roos M.
%Y Sérasset, Gilles
%Y Gkirtzou, Katerina
%Y Cochez, Michael
%Y Kalo, Jan-Christoph
%S Proceedings of the Knowledge Graphs and Large Language Models Workshop (KG-LLM) @ LREC26
%D 2026
%8 May
%I ELRA Language Resources Association (ELRA)
%C Palma, Mallorca (Spain)
%F kolk-etal-2026-evaluating
%X In a security environment that is growing more complex, large national organizations like the police rely on strategic frameworks to guide their decision-making. Frameworks like the Capability Based Planning (CBP) system are used to address this, but require a vast amount of information to function properly. A significant but underused store of information lies within an organization’s own internal flow of documents, like vision statements or annual reports. We tap into this flow by proposing a method to automatically extract relevant strategic entities and structuring them within a knowledge graph. We evaluate the performance of various Large Language Models (LLMs) on a corpus of policy excerpts from the Dutch National Police in extracting relevant strategic entities and linking them to core police capabilities. We employ the novel alternative annotator test (Alt-Test) to determine if an LLM can serve as a reliable substitute for a human domain expert on this highly subjective task. Our evaluation shows that while LLMs cannot fully replace human experts, they prove to be valuable support tools by frequently identifying the same strategic information as the annotators, successfully extracting core entities and linking them to predefined capabilities.
%R 10.63317/4pp744dg8ixs
%U https://aclanthology.org/2026.kallm-1.20/
%U https://doi.org/10.63317/4pp744dg8ixs
%P 210-220
Markdown (Informal)
[Evaluating Large Language Models for Strategic Knowledge Extraction in Capability-Based Planning](https://aclanthology.org/2026.kallm-1.20/) (Kolk et al., KaLLM 2026)
ACL