@inproceedings{monteiro-etal-2026-gutenbench,
title = "{G}uten{B}ench: A Taxonomy-Based Dataset for Evaluation of Information Retrieval Models",
author = "Monteiro, Marcio and
Senkin, Denys and
Ravichander, Abhilasha and
Kloft, Marius and
Fellenz, Sophie",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1342/",
pages = "30596--30613",
ISBN = "979-8-89176-459-0",
abstract = "While retrieval-augmented generation (RAG) relies on retrieved evidence to mitigate hallucination, its reliability is constrained by retrieval quality and diagnosing where retrieval fails remains difficult. Existing benchmarks typically report a single aggregate score over heterogeneous query sets and are often dominated by queries whose relevant passages can be identified through simple lexical overlap, concealing the failure modes that matter most in practice. We introduce GutenBench, a fine-grained, zero-shot evaluation dataset built around a four-level retrieval taxonomy: lexical, semantic, explicit aggregation, and latent conceptual retrieval. It pairs queries with relevance judgments adjudicated by subject-matter experts, enabling precise and controlled comparisons of retrievers. We evaluate several off-the-shelf retrievers in an asymmetric, English-to-target cross-lingual setting across 272 Bible translations (141 languages) and find that performance is strong on lexical and semantic queries, but degrades when models must aggregate information across passages or recognize conceptually relevant passages with no surface overlap."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="monteiro-etal-2026-gutenbench">
<titleInfo>
<title>GutenBench: A Taxonomy-Based Dataset for Evaluation of Information Retrieval Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Marcio</namePart>
<namePart type="family">Monteiro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Denys</namePart>
<namePart type="family">Senkin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abhilasha</namePart>
<namePart type="family">Ravichander</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marius</namePart>
<namePart type="family">Kloft</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sophie</namePart>
<namePart type="family">Fellenz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>While retrieval-augmented generation (RAG) relies on retrieved evidence to mitigate hallucination, its reliability is constrained by retrieval quality and diagnosing where retrieval fails remains difficult. Existing benchmarks typically report a single aggregate score over heterogeneous query sets and are often dominated by queries whose relevant passages can be identified through simple lexical overlap, concealing the failure modes that matter most in practice. We introduce GutenBench, a fine-grained, zero-shot evaluation dataset built around a four-level retrieval taxonomy: lexical, semantic, explicit aggregation, and latent conceptual retrieval. It pairs queries with relevance judgments adjudicated by subject-matter experts, enabling precise and controlled comparisons of retrievers. We evaluate several off-the-shelf retrievers in an asymmetric, English-to-target cross-lingual setting across 272 Bible translations (141 languages) and find that performance is strong on lexical and semantic queries, but degrades when models must aggregate information across passages or recognize conceptually relevant passages with no surface overlap.</abstract>
<identifier type="citekey">monteiro-etal-2026-gutenbench</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1342/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>30596</start>
<end>30613</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T GutenBench: A Taxonomy-Based Dataset for Evaluation of Information Retrieval Models
%A Monteiro, Marcio
%A Senkin, Denys
%A Ravichander, Abhilasha
%A Kloft, Marius
%A Fellenz, Sophie
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F monteiro-etal-2026-gutenbench
%X While retrieval-augmented generation (RAG) relies on retrieved evidence to mitigate hallucination, its reliability is constrained by retrieval quality and diagnosing where retrieval fails remains difficult. Existing benchmarks typically report a single aggregate score over heterogeneous query sets and are often dominated by queries whose relevant passages can be identified through simple lexical overlap, concealing the failure modes that matter most in practice. We introduce GutenBench, a fine-grained, zero-shot evaluation dataset built around a four-level retrieval taxonomy: lexical, semantic, explicit aggregation, and latent conceptual retrieval. It pairs queries with relevance judgments adjudicated by subject-matter experts, enabling precise and controlled comparisons of retrievers. We evaluate several off-the-shelf retrievers in an asymmetric, English-to-target cross-lingual setting across 272 Bible translations (141 languages) and find that performance is strong on lexical and semantic queries, but degrades when models must aggregate information across passages or recognize conceptually relevant passages with no surface overlap.
%U https://aclanthology.org/2026.emnlp-main.1342/
%P 30596-30613
Markdown (Informal)
[GutenBench: A Taxonomy-Based Dataset for Evaluation of Information Retrieval Models](https://aclanthology.org/2026.emnlp-main.1342/) (Monteiro et al., EMNLP 2026)
ACL