@inproceedings{alemany-etal-2026-zero,
title = "Zero-Shot Relation Classification with Large Language Models: Binary Prompting and Relevance Elicitation",
author = "Alemany, Sheila and
Sullivan-Pao, Danielle and
Simek, Olga",
editor = "Kalo, Jan-Christoph and
Nakashole, Ndapa and
Razniewski, Simon and
Suchanek, Fabian M. and
Vlachos, Andreas and
McCallum, Andrew",
booktitle = "Proceedings of the 11th Workshop on Automated Knowledge Base Construction ({AKBC} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.akbc-1.24/",
pages = "343--358",
ISBN = "979-8-89176-464-4",
abstract = "Relation classification (RC) transforms unstructured text into structured knowledge by identifying semantic relationships between entities. As natural language data volumes rapidly increase, manual knowledge extraction has become impractical. Existing zero-shot LLM-based approaches face limitations due to sparse RC task representation in instruction-tuning datasets and inherent biases toward certain relation types. To address these challenges, we propose Binary Relevance Elicitation (BRE), a true/false prompting framework that systematically queries an LLM for each candidate relation between two entities from a closed set of relation types. BRE incorporates relevance scoring to identify the most likely relation among those classified as ``true,'' reducing false positive predictions. We evaluate BRE using multiple open-source LLMs and achieve state-of-the-art zero-shot RC accuracy across the TACRED family of datasets. Our results demonstrate performance improvements with newer model releases that are independent of parameter count, suggesting information representation advances within LLMs drive zero-shot RC capabilities."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="alemany-etal-2026-zero">
<titleInfo>
<title>Zero-Shot Relation Classification with Large Language Models: Binary Prompting and Relevance Elicitation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sheila</namePart>
<namePart type="family">Alemany</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Danielle</namePart>
<namePart type="family">Sullivan-Pao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Olga</namePart>
<namePart type="family">Simek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 11th Workshop on Automated Knowledge Base Construction (AKBC 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jan-Christoph</namePart>
<namePart type="family">Kalo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ndapa</namePart>
<namePart type="family">Nakashole</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Razniewski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fabian</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Suchanek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andreas</namePart>
<namePart type="family">Vlachos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andrew</namePart>
<namePart type="family">McCallum</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-464-4</identifier>
</relatedItem>
<abstract>Relation classification (RC) transforms unstructured text into structured knowledge by identifying semantic relationships between entities. As natural language data volumes rapidly increase, manual knowledge extraction has become impractical. Existing zero-shot LLM-based approaches face limitations due to sparse RC task representation in instruction-tuning datasets and inherent biases toward certain relation types. To address these challenges, we propose Binary Relevance Elicitation (BRE), a true/false prompting framework that systematically queries an LLM for each candidate relation between two entities from a closed set of relation types. BRE incorporates relevance scoring to identify the most likely relation among those classified as “true,” reducing false positive predictions. We evaluate BRE using multiple open-source LLMs and achieve state-of-the-art zero-shot RC accuracy across the TACRED family of datasets. Our results demonstrate performance improvements with newer model releases that are independent of parameter count, suggesting information representation advances within LLMs drive zero-shot RC capabilities.</abstract>
<identifier type="citekey">alemany-etal-2026-zero</identifier>
<location>
<url>https://aclanthology.org/2026.akbc-1.24/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>343</start>
<end>358</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Zero-Shot Relation Classification with Large Language Models: Binary Prompting and Relevance Elicitation
%A Alemany, Sheila
%A Sullivan-Pao, Danielle
%A Simek, Olga
%Y Kalo, Jan-Christoph
%Y Nakashole, Ndapa
%Y Razniewski, Simon
%Y Suchanek, Fabian M.
%Y Vlachos, Andreas
%Y McCallum, Andrew
%S Proceedings of the 11th Workshop on Automated Knowledge Base Construction (AKBC 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-464-4
%F alemany-etal-2026-zero
%X Relation classification (RC) transforms unstructured text into structured knowledge by identifying semantic relationships between entities. As natural language data volumes rapidly increase, manual knowledge extraction has become impractical. Existing zero-shot LLM-based approaches face limitations due to sparse RC task representation in instruction-tuning datasets and inherent biases toward certain relation types. To address these challenges, we propose Binary Relevance Elicitation (BRE), a true/false prompting framework that systematically queries an LLM for each candidate relation between two entities from a closed set of relation types. BRE incorporates relevance scoring to identify the most likely relation among those classified as “true,” reducing false positive predictions. We evaluate BRE using multiple open-source LLMs and achieve state-of-the-art zero-shot RC accuracy across the TACRED family of datasets. Our results demonstrate performance improvements with newer model releases that are independent of parameter count, suggesting information representation advances within LLMs drive zero-shot RC capabilities.
%U https://aclanthology.org/2026.akbc-1.24/
%P 343-358
Markdown (Informal)
[Zero-Shot Relation Classification with Large Language Models: Binary Prompting and Relevance Elicitation](https://aclanthology.org/2026.akbc-1.24/) (Alemany et al., AKBC 2026)
ACL