@inproceedings{legrand-etal-2026-omop,
title = "An {OMOP}-Based Open-Source Text-to-{SQL} Benchmark Dataset",
author = "Legrand, Paul and
Noor, Kawsar and
Bhagwanani, Satyam and
Dobson, Richard J.",
editor = "Ben Abacha, Asma and
Bethard, Steven and
Bitterman, Danielle and
Naumann, Tristan and
Roberts, Kirk",
booktitle = "Proceedings of the 8th Workshop on Clinical Natural Language Processing (Clinical {NLP}) @ {LREC} 2026",
month = may,
year = "2026",
address = "Palma, Mallorca (Spain)",
publisher = "ELRA Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.clinicalnlp-1.40/",
doi = "10.63317/3hfefjmhhymh",
pages = "381--393",
abstract = "Access to electronic health record (EHR) warehouses is limited by SQL expertise and complex clinical schemas. We present an open-source OMOP Common Data Model text-to-SQL benchmark (CDM v5.4) with a safety contract: output one executable SQL statement or the abstention token ({\ensuremath{<}}NO{\_}SQL{\ensuremath{>}}) for unanswerable requests. Inputs are concept-normalized (entities as OMOP concept IDs) to decouple SQL generation from entity linking. We evaluate by executing predicted and reference queries on a synthetic OMOP PostgreSQL database, reporting Execution Accuracy (result equivalence) and a reliability score that rewards correct abstention and penalizes unsafe attempts. The dataset includes 6,690 paraphrases from 75 OMOP-adapted templates with leakage-resistant template/SQL-variation splits. LoRA-tuned Llama-3-8B-Instruct achieves 93.55{\%} execution accuracy with improved abstention reliability, while schema-injected baselines fail the contract. We release the dataset, splits, database dump, and a reproducible evaluation pipeline to support reliable clinical analytics assistants."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="legrand-etal-2026-omop">
<titleInfo>
<title>An OMOP-Based Open-Source Text-to-SQL Benchmark Dataset</title>
</titleInfo>
<name type="personal">
<namePart type="given">Paul</namePart>
<namePart type="family">Legrand</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kawsar</namePart>
<namePart type="family">Noor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Satyam</namePart>
<namePart type="family">Bhagwanani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Richard</namePart>
<namePart type="given">J</namePart>
<namePart type="family">Dobson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 8th Workshop on Clinical Natural Language Processing (Clinical NLP) @ LREC 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Asma</namePart>
<namePart type="family">Ben Abacha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Steven</namePart>
<namePart type="family">Bethard</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Danielle</namePart>
<namePart type="family">Bitterman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tristan</namePart>
<namePart type="family">Naumann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kirk</namePart>
<namePart type="family">Roberts</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma, Mallorca (Spain)</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Access to electronic health record (EHR) warehouses is limited by SQL expertise and complex clinical schemas. We present an open-source OMOP Common Data Model text-to-SQL benchmark (CDM v5.4) with a safety contract: output one executable SQL statement or the abstention token (\ensuremath<NO_SQL\ensuremath>) for unanswerable requests. Inputs are concept-normalized (entities as OMOP concept IDs) to decouple SQL generation from entity linking. We evaluate by executing predicted and reference queries on a synthetic OMOP PostgreSQL database, reporting Execution Accuracy (result equivalence) and a reliability score that rewards correct abstention and penalizes unsafe attempts. The dataset includes 6,690 paraphrases from 75 OMOP-adapted templates with leakage-resistant template/SQL-variation splits. LoRA-tuned Llama-3-8B-Instruct achieves 93.55% execution accuracy with improved abstention reliability, while schema-injected baselines fail the contract. We release the dataset, splits, database dump, and a reproducible evaluation pipeline to support reliable clinical analytics assistants.</abstract>
<identifier type="citekey">legrand-etal-2026-omop</identifier>
<identifier type="doi">10.63317/3hfefjmhhymh</identifier>
<location>
<url>https://aclanthology.org/2026.clinicalnlp-1.40/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>381</start>
<end>393</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T An OMOP-Based Open-Source Text-to-SQL Benchmark Dataset
%A Legrand, Paul
%A Noor, Kawsar
%A Bhagwanani, Satyam
%A Dobson, Richard J.
%Y Ben Abacha, Asma
%Y Bethard, Steven
%Y Bitterman, Danielle
%Y Naumann, Tristan
%Y Roberts, Kirk
%S Proceedings of the 8th Workshop on Clinical Natural Language Processing (Clinical NLP) @ LREC 2026
%D 2026
%8 May
%I ELRA Language Resources Association (ELRA)
%C Palma, Mallorca (Spain)
%F legrand-etal-2026-omop
%X Access to electronic health record (EHR) warehouses is limited by SQL expertise and complex clinical schemas. We present an open-source OMOP Common Data Model text-to-SQL benchmark (CDM v5.4) with a safety contract: output one executable SQL statement or the abstention token (\ensuremath<NO_SQL\ensuremath>) for unanswerable requests. Inputs are concept-normalized (entities as OMOP concept IDs) to decouple SQL generation from entity linking. We evaluate by executing predicted and reference queries on a synthetic OMOP PostgreSQL database, reporting Execution Accuracy (result equivalence) and a reliability score that rewards correct abstention and penalizes unsafe attempts. The dataset includes 6,690 paraphrases from 75 OMOP-adapted templates with leakage-resistant template/SQL-variation splits. LoRA-tuned Llama-3-8B-Instruct achieves 93.55% execution accuracy with improved abstention reliability, while schema-injected baselines fail the contract. We release the dataset, splits, database dump, and a reproducible evaluation pipeline to support reliable clinical analytics assistants.
%R 10.63317/3hfefjmhhymh
%U https://aclanthology.org/2026.clinicalnlp-1.40/
%U https://doi.org/10.63317/3hfefjmhhymh
%P 381-393
Markdown (Informal)
[An OMOP-Based Open-Source Text-to-SQL Benchmark Dataset](https://aclanthology.org/2026.clinicalnlp-1.40/) (Legrand et al., ClinicalNLP 2026)
ACL
- Paul Legrand, Kawsar Noor, Satyam Bhagwanani, and Richard J. Dobson. 2026. An OMOP-Based Open-Source Text-to-SQL Benchmark Dataset. In Proceedings of the 8th Workshop on Clinical Natural Language Processing (Clinical NLP) @ LREC 2026, pages 381–393, Palma, Mallorca (Spain). ELRA Language Resources Association (ELRA).