@inproceedings{georgiev-etal-2026-saber,
title = "{SABER}-Math: An Automated Reranking Benchmark for Mathematical Information Retrieval",
author = "Georgiev, Nikolay and
Drencheva, Maria and
Ibragimova, Kseniia and
Petrov, Ivo and
Dimitrov, Dimitar Iliev and
Vechev, Martin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2500/",
pages = "56772--56805",
ISBN = "979-8-89176-459-0",
abstract = "As agentic AI systems tackle more complex mathematical tasks, they increasingly rely on information retrieval (IR) to search problem databases, theorem libraries, and educational resources. However, choosing the right retriever remains difficult, as it is infeasible to isolate its effect on downstream performance. On the other hand, existing retrieval-specific benchmarks often fail to capture fine-grained mathematical relevance, penalizing systems that retrieve useful documents. We address this gap with SABER-Math, the first fully automated reranking benchmark for mathematical IR, built without expert annotation. Starting from 283K high-school-level problems with solutions, it builds challenging tasks in three steps: (i) LLMs extract concise solution summaries and topics for each problem; (ii) candidates are discovered per query using ontology-based topic similarity and lexical overlap between solution summaries; and (iii) a Swiss-style LLM preference tournament assigns every candidate a graded relevance rating. Over these judged candidate pools, we evaluate retrievers across all major paradigms, from lexical and math-specific systems to modern embedding, late-interaction, and reranking models. Reasoning embedders lead, followed by general-purpose models, both far outperforming classical and math-specific baselines. Yet, all still struggle in symbol-heavy domains such as Algebra and Calculus. Importantly, we show that generic IR benchmarks, such as MTEB, track mathematical performance broadly but are less discriminative among the strongest models, highlighting the need for domain-specific evaluation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="georgiev-etal-2026-saber">
<titleInfo>
<title>SABER-Math: An Automated Reranking Benchmark for Mathematical Information Retrieval</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nikolay</namePart>
<namePart type="family">Georgiev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maria</namePart>
<namePart type="family">Drencheva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kseniia</namePart>
<namePart type="family">Ibragimova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ivo</namePart>
<namePart type="family">Petrov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dimitar</namePart>
<namePart type="given">Iliev</namePart>
<namePart type="family">Dimitrov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Vechev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>As agentic AI systems tackle more complex mathematical tasks, they increasingly rely on information retrieval (IR) to search problem databases, theorem libraries, and educational resources. However, choosing the right retriever remains difficult, as it is infeasible to isolate its effect on downstream performance. On the other hand, existing retrieval-specific benchmarks often fail to capture fine-grained mathematical relevance, penalizing systems that retrieve useful documents. We address this gap with SABER-Math, the first fully automated reranking benchmark for mathematical IR, built without expert annotation. Starting from 283K high-school-level problems with solutions, it builds challenging tasks in three steps: (i) LLMs extract concise solution summaries and topics for each problem; (ii) candidates are discovered per query using ontology-based topic similarity and lexical overlap between solution summaries; and (iii) a Swiss-style LLM preference tournament assigns every candidate a graded relevance rating. Over these judged candidate pools, we evaluate retrievers across all major paradigms, from lexical and math-specific systems to modern embedding, late-interaction, and reranking models. Reasoning embedders lead, followed by general-purpose models, both far outperforming classical and math-specific baselines. Yet, all still struggle in symbol-heavy domains such as Algebra and Calculus. Importantly, we show that generic IR benchmarks, such as MTEB, track mathematical performance broadly but are less discriminative among the strongest models, highlighting the need for domain-specific evaluation.</abstract>
<identifier type="citekey">georgiev-etal-2026-saber</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2500/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>56772</start>
<end>56805</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SABER-Math: An Automated Reranking Benchmark for Mathematical Information Retrieval
%A Georgiev, Nikolay
%A Drencheva, Maria
%A Ibragimova, Kseniia
%A Petrov, Ivo
%A Dimitrov, Dimitar Iliev
%A Vechev, Martin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F georgiev-etal-2026-saber
%X As agentic AI systems tackle more complex mathematical tasks, they increasingly rely on information retrieval (IR) to search problem databases, theorem libraries, and educational resources. However, choosing the right retriever remains difficult, as it is infeasible to isolate its effect on downstream performance. On the other hand, existing retrieval-specific benchmarks often fail to capture fine-grained mathematical relevance, penalizing systems that retrieve useful documents. We address this gap with SABER-Math, the first fully automated reranking benchmark for mathematical IR, built without expert annotation. Starting from 283K high-school-level problems with solutions, it builds challenging tasks in three steps: (i) LLMs extract concise solution summaries and topics for each problem; (ii) candidates are discovered per query using ontology-based topic similarity and lexical overlap between solution summaries; and (iii) a Swiss-style LLM preference tournament assigns every candidate a graded relevance rating. Over these judged candidate pools, we evaluate retrievers across all major paradigms, from lexical and math-specific systems to modern embedding, late-interaction, and reranking models. Reasoning embedders lead, followed by general-purpose models, both far outperforming classical and math-specific baselines. Yet, all still struggle in symbol-heavy domains such as Algebra and Calculus. Importantly, we show that generic IR benchmarks, such as MTEB, track mathematical performance broadly but are less discriminative among the strongest models, highlighting the need for domain-specific evaluation.
%U https://aclanthology.org/2026.emnlp-main.2500/
%P 56772-56805
Markdown (Informal)
[SABER-Math: An Automated Reranking Benchmark for Mathematical Information Retrieval](https://aclanthology.org/2026.emnlp-main.2500/) (Georgiev et al., EMNLP 2026)
ACL