@inproceedings{gupta-2026-default,
title = "Default Zero-Shot {LLM} {API}s Over-Flag {I}ndian Financial {SMS}: A Policy-Aligned Benchmark and Template-Level Pre-Deployment Audit",
author = "Gupta, Sachin",
editor = "Wang, Yimu and
Choi, Yee Man and
Wu, Di and
Ouyang, Siqi and
Azadani, Mozhgan Nasr and
Fung, Yi R. (May)",
booktitle = "Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently ({G}round{LM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.groundlm-1.79/",
pages = "1124--1136",
ISBN = "979-8-89176-478-1",
abstract = "Generic ham/spam labels do not define a financial-fraud blocking policy. We present a 1,393-message policy-aligned Indian SMS benchmark in Hindi, Hinglish, and English-India, separating source labels, fraud/non-fraud/unclear adjudication, and allow/review/block actions. GPT-4o and Gemini helped adjudicate source-positive messages, so positive metrics are secondary. Our primary endpoint is false-positive rate (FPR) on 1,294 policy-negatives: 1,222 untouched source negatives plus 72 source positives converted to safe. Under a common minimal prompt, six LLM APIs exceed illustrative 1{\%} FPR gates on Bank and UPI templates. GPT-4o-mini records 77.3{\%} and 56.9{\%} FPR; a fine-tuned 270M XLM-R records 0.0{\%} on both, 0.4{\%} globally, and 0.980 recall. Policy definitions and seven examples reduce GPT-4o to 0.0{\%} Bank/UPI and 1.5{\%} global FPR at 0.949 recall; Claude reaches 1.8{\%} at 0.960 recall. Prompting changes which gate fails. GPT-4o P3 and the AND consensus meet the FPR gates at lower policy-label recall; the local model meets both FPR and recall gates on this benchmark. Because positive labels partly use GPT-4o/Gemini, recall and F1 are secondary, not an overall ranking. We release labels, prompts, parsed decisions, analysis code, and checkpoints without a production or unseen-institution claim."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="gupta-2026-default">
<titleInfo>
<title>Default Zero-Shot LLM APIs Over-Flag Indian Financial SMS: A Policy-Aligned Benchmark and Template-Level Pre-Deployment Audit</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sachin</namePart>
<namePart type="family">Gupta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yee</namePart>
<namePart type="given">Man</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siqi</namePart>
<namePart type="family">Ouyang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mozhgan</namePart>
<namePart type="given">Nasr</namePart>
<namePart type="family">Azadani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="given">R</namePart>
<namePart type="given">(May)</namePart>
<namePart type="family">Fung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-478-1</identifier>
</relatedItem>
<abstract>Generic ham/spam labels do not define a financial-fraud blocking policy. We present a 1,393-message policy-aligned Indian SMS benchmark in Hindi, Hinglish, and English-India, separating source labels, fraud/non-fraud/unclear adjudication, and allow/review/block actions. GPT-4o and Gemini helped adjudicate source-positive messages, so positive metrics are secondary. Our primary endpoint is false-positive rate (FPR) on 1,294 policy-negatives: 1,222 untouched source negatives plus 72 source positives converted to safe. Under a common minimal prompt, six LLM APIs exceed illustrative 1% FPR gates on Bank and UPI templates. GPT-4o-mini records 77.3% and 56.9% FPR; a fine-tuned 270M XLM-R records 0.0% on both, 0.4% globally, and 0.980 recall. Policy definitions and seven examples reduce GPT-4o to 0.0% Bank/UPI and 1.5% global FPR at 0.949 recall; Claude reaches 1.8% at 0.960 recall. Prompting changes which gate fails. GPT-4o P3 and the AND consensus meet the FPR gates at lower policy-label recall; the local model meets both FPR and recall gates on this benchmark. Because positive labels partly use GPT-4o/Gemini, recall and F1 are secondary, not an overall ranking. We release labels, prompts, parsed decisions, analysis code, and checkpoints without a production or unseen-institution claim.</abstract>
<identifier type="citekey">gupta-2026-default</identifier>
<location>
<url>https://aclanthology.org/2026.groundlm-1.79/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1124</start>
<end>1136</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Default Zero-Shot LLM APIs Over-Flag Indian Financial SMS: A Policy-Aligned Benchmark and Template-Level Pre-Deployment Audit
%A Gupta, Sachin
%Y Wang, Yimu
%Y Choi, Yee Man
%Y Wu, Di
%Y Ouyang, Siqi
%Y Azadani, Mozhgan Nasr
%Y Fung, Yi R. (May)
%S Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-478-1
%F gupta-2026-default
%X Generic ham/spam labels do not define a financial-fraud blocking policy. We present a 1,393-message policy-aligned Indian SMS benchmark in Hindi, Hinglish, and English-India, separating source labels, fraud/non-fraud/unclear adjudication, and allow/review/block actions. GPT-4o and Gemini helped adjudicate source-positive messages, so positive metrics are secondary. Our primary endpoint is false-positive rate (FPR) on 1,294 policy-negatives: 1,222 untouched source negatives plus 72 source positives converted to safe. Under a common minimal prompt, six LLM APIs exceed illustrative 1% FPR gates on Bank and UPI templates. GPT-4o-mini records 77.3% and 56.9% FPR; a fine-tuned 270M XLM-R records 0.0% on both, 0.4% globally, and 0.980 recall. Policy definitions and seven examples reduce GPT-4o to 0.0% Bank/UPI and 1.5% global FPR at 0.949 recall; Claude reaches 1.8% at 0.960 recall. Prompting changes which gate fails. GPT-4o P3 and the AND consensus meet the FPR gates at lower policy-label recall; the local model meets both FPR and recall gates on this benchmark. Because positive labels partly use GPT-4o/Gemini, recall and F1 are secondary, not an overall ranking. We release labels, prompts, parsed decisions, analysis code, and checkpoints without a production or unseen-institution claim.
%U https://aclanthology.org/2026.groundlm-1.79/
%P 1124-1136
Markdown (Informal)
[Default Zero-Shot LLM APIs Over-Flag Indian Financial SMS: A Policy-Aligned Benchmark and Template-Level Pre-Deployment Audit](https://aclanthology.org/2026.groundlm-1.79/) (Gupta, GroundLM 2026)
ACL