@inproceedings{lu-2026-relevance,
title = "When Relevance Is Not Evidence: A Controlled Diagnostic of Claim-Conditioned Support Ranking",
author = "Lu, Ke",
editor = "Kalo, Jan-Christoph and
Nakashole, Ndapa and
Razniewski, Simon and
Suchanek, Fabian M. and
Vlachos, Andreas and
McCallum, Andrew",
booktitle = "Proceedings of the 11th Workshop on Automated Knowledge Base Construction ({AKBC} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.akbc-1.7/",
pages = "88--93",
ISBN = "979-8-89176-464-4",
abstract = "Retrieval-augmented generation ranks passages by topical relevance or semantic similarity, but generation needs passages that actually support a target claim. We study when these targets come apart, framing the task as claim-conditioned support ranking over a constructed candidate shortlist: given a query, an explicit target claim, and a small fixed set of candidates, rank them by evidential support. We build a controlled, typed development set of 175 controlled contrastive examples spanning seven spurious-evidence cues, each pairing one supporting passage with one targeted non-support passage expressing the designated cue, plus neutral distractors. We evaluate with a corrected, tie-aware protocol that uses role-free evaluator-facing opaque identifiers and neutral tie handling. Tested query-only relevance rankers (lexical, dense, and a cross-encoder) yield tie-aware focal-pair error rates of roughly 18{--}34{\%}; in the BM25 cue decomposition, several strata have high error, including negation and relation directionality; a gold-claim NLI reference{---}which receives information the rankers do not{---}reduces but does not eliminate the error. We argue that evidence-ranking evaluation should explicitly condition on the claim whose support is assessed, and sketch a typed, claim-conditioned evaluation direction that separates relevance, support, and source attributes. This is a controlled diagnostic on development data: it does not estimate real-world prevalence or provide a deployable method."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lu-2026-relevance">
<titleInfo>
<title>When Relevance Is Not Evidence: A Controlled Diagnostic of Claim-Conditioned Support Ranking</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ke</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 11th Workshop on Automated Knowledge Base Construction (AKBC 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jan-Christoph</namePart>
<namePart type="family">Kalo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ndapa</namePart>
<namePart type="family">Nakashole</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Razniewski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fabian</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Suchanek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andreas</namePart>
<namePart type="family">Vlachos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andrew</namePart>
<namePart type="family">McCallum</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-464-4</identifier>
</relatedItem>
<abstract>Retrieval-augmented generation ranks passages by topical relevance or semantic similarity, but generation needs passages that actually support a target claim. We study when these targets come apart, framing the task as claim-conditioned support ranking over a constructed candidate shortlist: given a query, an explicit target claim, and a small fixed set of candidates, rank them by evidential support. We build a controlled, typed development set of 175 controlled contrastive examples spanning seven spurious-evidence cues, each pairing one supporting passage with one targeted non-support passage expressing the designated cue, plus neutral distractors. We evaluate with a corrected, tie-aware protocol that uses role-free evaluator-facing opaque identifiers and neutral tie handling. Tested query-only relevance rankers (lexical, dense, and a cross-encoder) yield tie-aware focal-pair error rates of roughly 18–34%; in the BM25 cue decomposition, several strata have high error, including negation and relation directionality; a gold-claim NLI reference—which receives information the rankers do not—reduces but does not eliminate the error. We argue that evidence-ranking evaluation should explicitly condition on the claim whose support is assessed, and sketch a typed, claim-conditioned evaluation direction that separates relevance, support, and source attributes. This is a controlled diagnostic on development data: it does not estimate real-world prevalence or provide a deployable method.</abstract>
<identifier type="citekey">lu-2026-relevance</identifier>
<location>
<url>https://aclanthology.org/2026.akbc-1.7/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>88</start>
<end>93</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Relevance Is Not Evidence: A Controlled Diagnostic of Claim-Conditioned Support Ranking
%A Lu, Ke
%Y Kalo, Jan-Christoph
%Y Nakashole, Ndapa
%Y Razniewski, Simon
%Y Suchanek, Fabian M.
%Y Vlachos, Andreas
%Y McCallum, Andrew
%S Proceedings of the 11th Workshop on Automated Knowledge Base Construction (AKBC 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-464-4
%F lu-2026-relevance
%X Retrieval-augmented generation ranks passages by topical relevance or semantic similarity, but generation needs passages that actually support a target claim. We study when these targets come apart, framing the task as claim-conditioned support ranking over a constructed candidate shortlist: given a query, an explicit target claim, and a small fixed set of candidates, rank them by evidential support. We build a controlled, typed development set of 175 controlled contrastive examples spanning seven spurious-evidence cues, each pairing one supporting passage with one targeted non-support passage expressing the designated cue, plus neutral distractors. We evaluate with a corrected, tie-aware protocol that uses role-free evaluator-facing opaque identifiers and neutral tie handling. Tested query-only relevance rankers (lexical, dense, and a cross-encoder) yield tie-aware focal-pair error rates of roughly 18–34%; in the BM25 cue decomposition, several strata have high error, including negation and relation directionality; a gold-claim NLI reference—which receives information the rankers do not—reduces but does not eliminate the error. We argue that evidence-ranking evaluation should explicitly condition on the claim whose support is assessed, and sketch a typed, claim-conditioned evaluation direction that separates relevance, support, and source attributes. This is a controlled diagnostic on development data: it does not estimate real-world prevalence or provide a deployable method.
%U https://aclanthology.org/2026.akbc-1.7/
%P 88-93
Markdown (Informal)
[When Relevance Is Not Evidence: A Controlled Diagnostic of Claim-Conditioned Support Ranking](https://aclanthology.org/2026.akbc-1.7/) (Lu, AKBC 2026)
ACL