@inproceedings{more-jadhav-2026-retrieve,
title = "Retrieve, Locate, Generate: An Oracle-Substitution Diagnostic for Literature-Grounded {QA}",
author = "More, Siddhesh and
Jadhav, Kunal",
editor = "Wang, Yimu and
Choi, Yee Man and
Wu, Di and
Ouyang, Siqi and
Azadani, Mozhgan Nasr and
Fung, Yi R. (May)",
booktitle = "Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently ({G}round{LM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.groundlm-1.57/",
pages = "767--778",
ISBN = "979-8-89176-478-1",
abstract = "Literature-grounded question answering requires a system to retrieve the right paper, locate the right evidence within it, and generate an answer faithful to that evidence. These are three separate stages, each with its own failure mode. We present a stage-wise diagnostic study of this pipeline on LitTraceQA, a shared task requiring paper retrieval, coarse evidence localization (table/figure/page), and structured answer generation. Our pipeline combines cross-model hypothesis diversity for retrieval, a filter-then-rank evidence localizer (caption-pattern match, embedding similarity, evidence-type gate), and cross-model semantic entropy for generation-time answer aggregation and confidence estimation. We use an oracle-substitution ablation ladder, sequentially handing the pipeline the gold paper and then the gold evidence value, and reading off the resulting change in each stage{'}s own metric. These are sequential oracle-intervention gains, not an additive error decomposition: the stages are measured with different quantities and are not independent. Read that way, the ladder points at evidence localization as the dominant remediable bottleneck {---} multiple-choice accuracy rises from 0.42 with predicted evidence to 0.76 with the gold evidence value, and end-to-end accuracy stays near 0.29 even on the questions where retrieval already returned every gold paper. We also test whether cheap uncertainty signals computed at retrieval time predict which stage will fail before it does. Along the way we surface two infrastructure-level findings with direct implications for anyone building literature-grounded QA systems. First, evidence page numbers in this and similar benchmarks are keyed to a paper{'}s canonical published version rather than its arXiv preprint; a real failure mode when a paper has been substantively revised, though a systematic paired ablation across 45 questions finds it is an occasional rather than pervasive hazard in this pool. Second, a substantial fraction of papers hosted on OpenReview (ICLR/ICML/NeurIPS) are unreachable by ordinary scripted HTTP requests due to a JS bot-challenge, a venue-correlated failure mode unrelated to any modeling choice. We report our diagnostic protocol, quantitative results with bootstrap confidence intervals given the benchmark{'}s small public dev set, and a qualitative failure taxonomy, framing this as a resource and lessons-learned contribution rather than a leaderboard claim. All multiple-choice numbers are a best-case diagnostic condition: they assume access to the gold option text, which a genuine blind submission would not have."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="more-jadhav-2026-retrieve">
<titleInfo>
<title>Retrieve, Locate, Generate: An Oracle-Substitution Diagnostic for Literature-Grounded QA</title>
</titleInfo>
<name type="personal">
<namePart type="given">Siddhesh</namePart>
<namePart type="family">More</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kunal</namePart>
<namePart type="family">Jadhav</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yee</namePart>
<namePart type="given">Man</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siqi</namePart>
<namePart type="family">Ouyang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mozhgan</namePart>
<namePart type="given">Nasr</namePart>
<namePart type="family">Azadani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="given">R</namePart>
<namePart type="given">(May)</namePart>
<namePart type="family">Fung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-478-1</identifier>
</relatedItem>
<abstract>Literature-grounded question answering requires a system to retrieve the right paper, locate the right evidence within it, and generate an answer faithful to that evidence. These are three separate stages, each with its own failure mode. We present a stage-wise diagnostic study of this pipeline on LitTraceQA, a shared task requiring paper retrieval, coarse evidence localization (table/figure/page), and structured answer generation. Our pipeline combines cross-model hypothesis diversity for retrieval, a filter-then-rank evidence localizer (caption-pattern match, embedding similarity, evidence-type gate), and cross-model semantic entropy for generation-time answer aggregation and confidence estimation. We use an oracle-substitution ablation ladder, sequentially handing the pipeline the gold paper and then the gold evidence value, and reading off the resulting change in each stage’s own metric. These are sequential oracle-intervention gains, not an additive error decomposition: the stages are measured with different quantities and are not independent. Read that way, the ladder points at evidence localization as the dominant remediable bottleneck — multiple-choice accuracy rises from 0.42 with predicted evidence to 0.76 with the gold evidence value, and end-to-end accuracy stays near 0.29 even on the questions where retrieval already returned every gold paper. We also test whether cheap uncertainty signals computed at retrieval time predict which stage will fail before it does. Along the way we surface two infrastructure-level findings with direct implications for anyone building literature-grounded QA systems. First, evidence page numbers in this and similar benchmarks are keyed to a paper’s canonical published version rather than its arXiv preprint; a real failure mode when a paper has been substantively revised, though a systematic paired ablation across 45 questions finds it is an occasional rather than pervasive hazard in this pool. Second, a substantial fraction of papers hosted on OpenReview (ICLR/ICML/NeurIPS) are unreachable by ordinary scripted HTTP requests due to a JS bot-challenge, a venue-correlated failure mode unrelated to any modeling choice. We report our diagnostic protocol, quantitative results with bootstrap confidence intervals given the benchmark’s small public dev set, and a qualitative failure taxonomy, framing this as a resource and lessons-learned contribution rather than a leaderboard claim. All multiple-choice numbers are a best-case diagnostic condition: they assume access to the gold option text, which a genuine blind submission would not have.</abstract>
<identifier type="citekey">more-jadhav-2026-retrieve</identifier>
<location>
<url>https://aclanthology.org/2026.groundlm-1.57/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>767</start>
<end>778</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Retrieve, Locate, Generate: An Oracle-Substitution Diagnostic for Literature-Grounded QA
%A More, Siddhesh
%A Jadhav, Kunal
%Y Wang, Yimu
%Y Choi, Yee Man
%Y Wu, Di
%Y Ouyang, Siqi
%Y Azadani, Mozhgan Nasr
%Y Fung, Yi R. (May)
%S Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-478-1
%F more-jadhav-2026-retrieve
%X Literature-grounded question answering requires a system to retrieve the right paper, locate the right evidence within it, and generate an answer faithful to that evidence. These are three separate stages, each with its own failure mode. We present a stage-wise diagnostic study of this pipeline on LitTraceQA, a shared task requiring paper retrieval, coarse evidence localization (table/figure/page), and structured answer generation. Our pipeline combines cross-model hypothesis diversity for retrieval, a filter-then-rank evidence localizer (caption-pattern match, embedding similarity, evidence-type gate), and cross-model semantic entropy for generation-time answer aggregation and confidence estimation. We use an oracle-substitution ablation ladder, sequentially handing the pipeline the gold paper and then the gold evidence value, and reading off the resulting change in each stage’s own metric. These are sequential oracle-intervention gains, not an additive error decomposition: the stages are measured with different quantities and are not independent. Read that way, the ladder points at evidence localization as the dominant remediable bottleneck — multiple-choice accuracy rises from 0.42 with predicted evidence to 0.76 with the gold evidence value, and end-to-end accuracy stays near 0.29 even on the questions where retrieval already returned every gold paper. We also test whether cheap uncertainty signals computed at retrieval time predict which stage will fail before it does. Along the way we surface two infrastructure-level findings with direct implications for anyone building literature-grounded QA systems. First, evidence page numbers in this and similar benchmarks are keyed to a paper’s canonical published version rather than its arXiv preprint; a real failure mode when a paper has been substantively revised, though a systematic paired ablation across 45 questions finds it is an occasional rather than pervasive hazard in this pool. Second, a substantial fraction of papers hosted on OpenReview (ICLR/ICML/NeurIPS) are unreachable by ordinary scripted HTTP requests due to a JS bot-challenge, a venue-correlated failure mode unrelated to any modeling choice. We report our diagnostic protocol, quantitative results with bootstrap confidence intervals given the benchmark’s small public dev set, and a qualitative failure taxonomy, framing this as a resource and lessons-learned contribution rather than a leaderboard claim. All multiple-choice numbers are a best-case diagnostic condition: they assume access to the gold option text, which a genuine blind submission would not have.
%U https://aclanthology.org/2026.groundlm-1.57/
%P 767-778
Markdown (Informal)
[Retrieve, Locate, Generate: An Oracle-Substitution Diagnostic for Literature-Grounded QA](https://aclanthology.org/2026.groundlm-1.57/) (More & Jadhav, GroundLM 2026)
ACL