@inproceedings{zhou-etal-2026-automated,
title = "An Automated Pipeline for Provably Retrieval-Dependent Benchmark Construction over Dynamic Knowledge",
author = "Zhou, Heng and
Yu, Ao and
Fan, Yuchen and
Kang, Li and
Shi, Jianing and
Zhang, Yongting and
Fan, Yutao and
He, Tiancheng and
Lin, Yibing and
Geng, Hejia and
Wu, Yuhao and
Song, Xiufeng and
Zhang, Zhemeng and
Huang, Songtao and
Qin, Yiran and
Zhang, Wenlong and
Bai, Lei and
Yin, Zhenfei",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1907/",
pages = "40633--40648",
ISBN = "979-8-89176-462-0",
abstract = "Static benchmarks often conflate memorization with reasoning, failing to capture the dynamic nature of world knowledge. We present LiveSearchBench, a scalable pipeline that automatically constructs snapshot-grounded benchmarks from knowledge graph differentials, in which every answer is unique and SPARQL-verifiable against a target snapshot. Unlike prior template-based approaches, our method leverages topology-aware synthesis to generate multi-constraint compositional questions with unique answers guaranteed via SPARQL validation. Evaluating 1,000 questions across three difficulty levels on both open-weight and frontier models, we expose a pronounced Recency Gap on facts that post-date a model{'}s training cutoff, particularly for compositional queries; retrieval dependence is thus an empirical property we measure, not a formal guarantee we claim. Through Oracle and Wikidata-as-corpus analyses, we disentangle dual failure modes: retrieval coverage limitations for single-hop questions and reasoning failures for compositional queries even with ground-truth evidence. LiveSearchBench provides both a rigorous construction methodology and a continuously updatable testbed for evaluating evidence integration under evolving knowledge."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhou-etal-2026-automated">
<titleInfo>
<title>An Automated Pipeline for Provably Retrieval-Dependent Benchmark Construction over Dynamic Knowledge</title>
</titleInfo>
<name type="personal">
<namePart type="given">Heng</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ao</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuchen</namePart>
<namePart type="family">Fan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Li</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jianing</namePart>
<namePart type="family">Shi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yongting</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yutao</namePart>
<namePart type="family">Fan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tiancheng</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yibing</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hejia</namePart>
<namePart type="family">Geng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuhao</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiufeng</namePart>
<namePart type="family">Song</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhemeng</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Songtao</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yiran</namePart>
<namePart type="family">Qin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenlong</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lei</namePart>
<namePart type="family">Bai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhenfei</namePart>
<namePart type="family">Yin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Static benchmarks often conflate memorization with reasoning, failing to capture the dynamic nature of world knowledge. We present LiveSearchBench, a scalable pipeline that automatically constructs snapshot-grounded benchmarks from knowledge graph differentials, in which every answer is unique and SPARQL-verifiable against a target snapshot. Unlike prior template-based approaches, our method leverages topology-aware synthesis to generate multi-constraint compositional questions with unique answers guaranteed via SPARQL validation. Evaluating 1,000 questions across three difficulty levels on both open-weight and frontier models, we expose a pronounced Recency Gap on facts that post-date a model’s training cutoff, particularly for compositional queries; retrieval dependence is thus an empirical property we measure, not a formal guarantee we claim. Through Oracle and Wikidata-as-corpus analyses, we disentangle dual failure modes: retrieval coverage limitations for single-hop questions and reasoning failures for compositional queries even with ground-truth evidence. LiveSearchBench provides both a rigorous construction methodology and a continuously updatable testbed for evaluating evidence integration under evolving knowledge.</abstract>
<identifier type="citekey">zhou-etal-2026-automated</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1907/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>40633</start>
<end>40648</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T An Automated Pipeline for Provably Retrieval-Dependent Benchmark Construction over Dynamic Knowledge
%A Zhou, Heng
%A Yu, Ao
%A Fan, Yuchen
%A Kang, Li
%A Shi, Jianing
%A Zhang, Yongting
%A Fan, Yutao
%A He, Tiancheng
%A Lin, Yibing
%A Geng, Hejia
%A Wu, Yuhao
%A Song, Xiufeng
%A Zhang, Zhemeng
%A Huang, Songtao
%A Qin, Yiran
%A Zhang, Wenlong
%A Bai, Lei
%A Yin, Zhenfei
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F zhou-etal-2026-automated
%X Static benchmarks often conflate memorization with reasoning, failing to capture the dynamic nature of world knowledge. We present LiveSearchBench, a scalable pipeline that automatically constructs snapshot-grounded benchmarks from knowledge graph differentials, in which every answer is unique and SPARQL-verifiable against a target snapshot. Unlike prior template-based approaches, our method leverages topology-aware synthesis to generate multi-constraint compositional questions with unique answers guaranteed via SPARQL validation. Evaluating 1,000 questions across three difficulty levels on both open-weight and frontier models, we expose a pronounced Recency Gap on facts that post-date a model’s training cutoff, particularly for compositional queries; retrieval dependence is thus an empirical property we measure, not a formal guarantee we claim. Through Oracle and Wikidata-as-corpus analyses, we disentangle dual failure modes: retrieval coverage limitations for single-hop questions and reasoning failures for compositional queries even with ground-truth evidence. LiveSearchBench provides both a rigorous construction methodology and a continuously updatable testbed for evaluating evidence integration under evolving knowledge.
%U https://aclanthology.org/2026.findings-emnlp.1907/
%P 40633-40648
Markdown (Informal)
[An Automated Pipeline for Provably Retrieval-Dependent Benchmark Construction over Dynamic Knowledge](https://aclanthology.org/2026.findings-emnlp.1907/) (Zhou et al., Findings 2026)
ACL
- Heng Zhou, Ao Yu, Yuchen Fan, Li Kang, Jianing Shi, Yongting Zhang, Yutao Fan, Tiancheng He, Yibing Lin, Hejia Geng, Yuhao Wu, Xiufeng Song, Zhemeng Zhang, Songtao Huang, Yiran Qin, Wenlong Zhang, Lei Bai, and Zhenfei Yin. 2026. An Automated Pipeline for Provably Retrieval-Dependent Benchmark Construction over Dynamic Knowledge. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 40633–40648, Budapest, Hungary. Association for Computational Linguistics.