@inproceedings{hill-etal-2026-bird,
title = "{BIRD}-Persona: A Persona-Aware Benchmark for {SQL} and pandas Analytics Code Generation",
author = "Hill, Davin and
Chorasiya, Pawan and
Leelar, Bhawani Shankar and
Avram, Oren and
Tillman, Robert E. and
Soliman, Tamer",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.138/",
pages = "2311--2332",
ISBN = "979-8-89176-460-6",
abstract = "Production text-to-code systems increasingly use LLMs to translate natural-language analytics requests into executable SQL and pandas code, serving both business users, who describe desired results in application terms, and technical users, who name tables and columns directly. Existing Text-to-SQL and Text-to-Python benchmarks provide coverage, but they do not isolate when SQL and pandas performance diverge. We introduce BIRD-Persona, a persona-stratified extension of BIRD-Python that adds novice/expert rewrites (business-style and technical-style phrasing) to tasks with matched SQL and pandas ground truth, plus schema variants and task-structure annotations. Across 1,534 tasks, nine models, and 245,592 evaluations, BIRD-Persona reproduces the previously reported SQL-over-Python gap: SQL outperforms pandas for eight of nine systems. The gap is not uniform: expert, schema-explicit prompts improve both paradigms but help SQL more, schema representations have mixed model-dependent effects, and selected-column burden widens the gap while subqueries do not. By making user phrasing a controlled benchmark dimension, BIRD-Persona provides a foundation for persona-aware and paradigm-aware evaluation of natural-language analytics systems."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hill-etal-2026-bird">
<titleInfo>
<title>BIRD-Persona: A Persona-Aware Benchmark for SQL and pandas Analytics Code Generation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Davin</namePart>
<namePart type="family">Hill</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pawan</namePart>
<namePart type="family">Chorasiya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bhawani</namePart>
<namePart type="given">Shankar</namePart>
<namePart type="family">Leelar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Oren</namePart>
<namePart type="family">Avram</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Robert</namePart>
<namePart type="given">E</namePart>
<namePart type="family">Tillman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tamer</namePart>
<namePart type="family">Soliman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>Production text-to-code systems increasingly use LLMs to translate natural-language analytics requests into executable SQL and pandas code, serving both business users, who describe desired results in application terms, and technical users, who name tables and columns directly. Existing Text-to-SQL and Text-to-Python benchmarks provide coverage, but they do not isolate when SQL and pandas performance diverge. We introduce BIRD-Persona, a persona-stratified extension of BIRD-Python that adds novice/expert rewrites (business-style and technical-style phrasing) to tasks with matched SQL and pandas ground truth, plus schema variants and task-structure annotations. Across 1,534 tasks, nine models, and 245,592 evaluations, BIRD-Persona reproduces the previously reported SQL-over-Python gap: SQL outperforms pandas for eight of nine systems. The gap is not uniform: expert, schema-explicit prompts improve both paradigms but help SQL more, schema representations have mixed model-dependent effects, and selected-column burden widens the gap while subqueries do not. By making user phrasing a controlled benchmark dimension, BIRD-Persona provides a foundation for persona-aware and paradigm-aware evaluation of natural-language analytics systems.</abstract>
<identifier type="citekey">hill-etal-2026-bird</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.138/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>2311</start>
<end>2332</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T BIRD-Persona: A Persona-Aware Benchmark for SQL and pandas Analytics Code Generation
%A Hill, Davin
%A Chorasiya, Pawan
%A Leelar, Bhawani Shankar
%A Avram, Oren
%A Tillman, Robert E.
%A Soliman, Tamer
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F hill-etal-2026-bird
%X Production text-to-code systems increasingly use LLMs to translate natural-language analytics requests into executable SQL and pandas code, serving both business users, who describe desired results in application terms, and technical users, who name tables and columns directly. Existing Text-to-SQL and Text-to-Python benchmarks provide coverage, but they do not isolate when SQL and pandas performance diverge. We introduce BIRD-Persona, a persona-stratified extension of BIRD-Python that adds novice/expert rewrites (business-style and technical-style phrasing) to tasks with matched SQL and pandas ground truth, plus schema variants and task-structure annotations. Across 1,534 tasks, nine models, and 245,592 evaluations, BIRD-Persona reproduces the previously reported SQL-over-Python gap: SQL outperforms pandas for eight of nine systems. The gap is not uniform: expert, schema-explicit prompts improve both paradigms but help SQL more, schema representations have mixed model-dependent effects, and selected-column burden widens the gap while subqueries do not. By making user phrasing a controlled benchmark dimension, BIRD-Persona provides a foundation for persona-aware and paradigm-aware evaluation of natural-language analytics systems.
%U https://aclanthology.org/2026.emnlp-industry.138/
%P 2311-2332
Markdown (Informal)
[BIRD-Persona: A Persona-Aware Benchmark for SQL and pandas Analytics Code Generation](https://aclanthology.org/2026.emnlp-industry.138/) (Hill et al., EMNLP 2026)
ACL