@inproceedings{aluguvelly-2026-sql,
title = "Same {SQL}, Different Decision: Aggregate Risk Hides Cross-Lingual Instability in Text-to-{SQL} Verification",
author = "Aluguvelly, Avinash Goutham",
editor = "Ranaldi, Leonardo and
Pucci, Giulia and
Ranaldi, Federico and
Valentino, Marco and
Birch, Alexandra and
Zanzotto, Fabio Massimo",
booktitle = "The Proceedings of the Workshop on Open Reasoning Across Cultures Languages ({ORACLE})",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.oracle-main.6/",
pages = "84--98",
ISBN = "979-8-89176-483-5",
abstract = "Text-to-SQL systems that use an LLM verifier typically calibrate its execute-or-defer threshold on English requests. We ask whether that threshold still makes the same decisions when the user writes in another language. Our evaluation is paired: the candidate SQL, the schema, the database, and the execution-derived correctness label are held fixed, and only the question changes, across seven languages and two open-weight verifiers. The verifiers rank candidates almost as well in every language, yet 14{\%} to 27{\%} of identical SQL receives a different execute-or-defer decision. The usual aggregates hide this. Selective risk stays near its English value because promotions and demotions offset, and coverage flags only the languages where the shift is one-sided. Underneath, both harms occur: correct queries are needlessly deferred, and incorrect queries that English withheld are executed; under Vietnamese, Llama executes 16.9{\%} of all incorrect candidates, 22.9{\%} of those it had deferred in English. Score-level corrections that equalize coverage barely change which queries run. Translating the request into English before verification helps more, lowering the flip rate and unsafe promotions on both verifiers, though it does not restore the English policy. Audits of multilingual verifiers should therefore report decision transitions, not only aggregate coverage and risk."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="aluguvelly-2026-sql">
<titleInfo>
<title>Same SQL, Different Decision: Aggregate Risk Hides Cross-Lingual Instability in Text-to-SQL Verification</title>
</titleInfo>
<name type="personal">
<namePart type="given">Avinash</namePart>
<namePart type="given">Goutham</namePart>
<namePart type="family">Aluguvelly</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>The Proceedings of the Workshop on Open Reasoning Across Cultures Languages (ORACLE)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leonardo</namePart>
<namePart type="family">Ranaldi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Giulia</namePart>
<namePart type="family">Pucci</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Federico</namePart>
<namePart type="family">Ranaldi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marco</namePart>
<namePart type="family">Valentino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexandra</namePart>
<namePart type="family">Birch</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fabio</namePart>
<namePart type="given">Massimo</namePart>
<namePart type="family">Zanzotto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-483-5</identifier>
</relatedItem>
<abstract>Text-to-SQL systems that use an LLM verifier typically calibrate its execute-or-defer threshold on English requests. We ask whether that threshold still makes the same decisions when the user writes in another language. Our evaluation is paired: the candidate SQL, the schema, the database, and the execution-derived correctness label are held fixed, and only the question changes, across seven languages and two open-weight verifiers. The verifiers rank candidates almost as well in every language, yet 14% to 27% of identical SQL receives a different execute-or-defer decision. The usual aggregates hide this. Selective risk stays near its English value because promotions and demotions offset, and coverage flags only the languages where the shift is one-sided. Underneath, both harms occur: correct queries are needlessly deferred, and incorrect queries that English withheld are executed; under Vietnamese, Llama executes 16.9% of all incorrect candidates, 22.9% of those it had deferred in English. Score-level corrections that equalize coverage barely change which queries run. Translating the request into English before verification helps more, lowering the flip rate and unsafe promotions on both verifiers, though it does not restore the English policy. Audits of multilingual verifiers should therefore report decision transitions, not only aggregate coverage and risk.</abstract>
<identifier type="citekey">aluguvelly-2026-sql</identifier>
<location>
<url>https://aclanthology.org/2026.oracle-main.6/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>84</start>
<end>98</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Same SQL, Different Decision: Aggregate Risk Hides Cross-Lingual Instability in Text-to-SQL Verification
%A Aluguvelly, Avinash Goutham
%Y Ranaldi, Leonardo
%Y Pucci, Giulia
%Y Ranaldi, Federico
%Y Valentino, Marco
%Y Birch, Alexandra
%Y Zanzotto, Fabio Massimo
%S The Proceedings of the Workshop on Open Reasoning Across Cultures Languages (ORACLE)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-483-5
%F aluguvelly-2026-sql
%X Text-to-SQL systems that use an LLM verifier typically calibrate its execute-or-defer threshold on English requests. We ask whether that threshold still makes the same decisions when the user writes in another language. Our evaluation is paired: the candidate SQL, the schema, the database, and the execution-derived correctness label are held fixed, and only the question changes, across seven languages and two open-weight verifiers. The verifiers rank candidates almost as well in every language, yet 14% to 27% of identical SQL receives a different execute-or-defer decision. The usual aggregates hide this. Selective risk stays near its English value because promotions and demotions offset, and coverage flags only the languages where the shift is one-sided. Underneath, both harms occur: correct queries are needlessly deferred, and incorrect queries that English withheld are executed; under Vietnamese, Llama executes 16.9% of all incorrect candidates, 22.9% of those it had deferred in English. Score-level corrections that equalize coverage barely change which queries run. Translating the request into English before verification helps more, lowering the flip rate and unsafe promotions on both verifiers, though it does not restore the English policy. Audits of multilingual verifiers should therefore report decision transitions, not only aggregate coverage and risk.
%U https://aclanthology.org/2026.oracle-main.6/
%P 84-98
Markdown (Informal)
[Same SQL, Different Decision: Aggregate Risk Hides Cross-Lingual Instability in Text-to-SQL Verification](https://aclanthology.org/2026.oracle-main.6/) (Aluguvelly, ORACLE 2026)
ACL