@inproceedings{vo-nguyen-2026-holmes,
title = "Holmes-{SQL}: Benchmarking {LLM} Agents on Evidence-Aware Investigative Reasoning over Structured Databases",
author = "Vo, Hoang-An and
Nguyen, Cam-Tu",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1267/",
pages = "28868--28893",
ISBN = "979-8-89176-459-0",
abstract = "Existing SQL-based benchmarks usually assume that retrieved records are equally informative. This assumption is limiting for investigative domains, where truthful records may differ substantially in inferential strength. To address this gap, we introduce Holmes-SQL, the first benchmark for urban crime investigation over relational databases. Each case begins with partial cues and requires agents to reconstruct the crime scene, victim, suspect pool, and culprit through database interaction. Holmes-SQL combines state uncertainty, unequal evidential strength, and fallacy-aligned suspect construction to evaluate evidence-aware investigative reasoning. Experiments show that agents achieve over 97{\%} SQL execution success, yet the best solve rate reaches only 49{\%}, highlighting a distinct challenge for evidence-aware investigative reasoning. A further comparison shows that agents consistently underperform when investigating compared with reasoning over all evidence provided upfront, highlighting the added challenge of investigation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="vo-nguyen-2026-holmes">
<titleInfo>
<title>Holmes-SQL: Benchmarking LLM Agents on Evidence-Aware Investigative Reasoning over Structured Databases</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hoang-An</namePart>
<namePart type="family">Vo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cam-Tu</namePart>
<namePart type="family">Nguyen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Existing SQL-based benchmarks usually assume that retrieved records are equally informative. This assumption is limiting for investigative domains, where truthful records may differ substantially in inferential strength. To address this gap, we introduce Holmes-SQL, the first benchmark for urban crime investigation over relational databases. Each case begins with partial cues and requires agents to reconstruct the crime scene, victim, suspect pool, and culprit through database interaction. Holmes-SQL combines state uncertainty, unequal evidential strength, and fallacy-aligned suspect construction to evaluate evidence-aware investigative reasoning. Experiments show that agents achieve over 97% SQL execution success, yet the best solve rate reaches only 49%, highlighting a distinct challenge for evidence-aware investigative reasoning. A further comparison shows that agents consistently underperform when investigating compared with reasoning over all evidence provided upfront, highlighting the added challenge of investigation.</abstract>
<identifier type="citekey">vo-nguyen-2026-holmes</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1267/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>28868</start>
<end>28893</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Holmes-SQL: Benchmarking LLM Agents on Evidence-Aware Investigative Reasoning over Structured Databases
%A Vo, Hoang-An
%A Nguyen, Cam-Tu
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F vo-nguyen-2026-holmes
%X Existing SQL-based benchmarks usually assume that retrieved records are equally informative. This assumption is limiting for investigative domains, where truthful records may differ substantially in inferential strength. To address this gap, we introduce Holmes-SQL, the first benchmark for urban crime investigation over relational databases. Each case begins with partial cues and requires agents to reconstruct the crime scene, victim, suspect pool, and culprit through database interaction. Holmes-SQL combines state uncertainty, unequal evidential strength, and fallacy-aligned suspect construction to evaluate evidence-aware investigative reasoning. Experiments show that agents achieve over 97% SQL execution success, yet the best solve rate reaches only 49%, highlighting a distinct challenge for evidence-aware investigative reasoning. A further comparison shows that agents consistently underperform when investigating compared with reasoning over all evidence provided upfront, highlighting the added challenge of investigation.
%U https://aclanthology.org/2026.emnlp-main.1267/
%P 28868-28893
Markdown (Informal)
[Holmes-SQL: Benchmarking LLM Agents on Evidence-Aware Investigative Reasoning over Structured Databases](https://aclanthology.org/2026.emnlp-main.1267/) (Vo & Nguyen, EMNLP 2026)
ACL