@inproceedings{shah-etal-2026-persuading,
title = "Persuading the Judge: Adversarial Robustness and {RAG} Mitigation in {LLM}-Based Legal Citation Verification",
author = "Shah, Rohann Rahul and
Jawagal, Rishabh Kartik and
Srinivasa, Gowri",
editor = "Aletras, Nikolaos and
Chalkidis, Ilias and
Barrett, Leslie and
Goanț{\u{a}}, C{\u{a}}t{\u{a}}lina and
Preoțiuc-Pietro, Daniel and
Spanakis, Gerasimos and
Xu, Shanshan",
booktitle = "Proceedings of the Natural Legal Language Processing Workshop 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.nllp-1.6/",
pages = "42--54",
ISBN = "979-8-89176-494-1",
abstract = "Automated legal citation verification systems are critical for ensuring the factual integrity of AI-assisted legal research and drafting, but large language models (LLMs) serving as verifiers remain vulnerable to adversarial persuasion. We introduce a three-class benchmark dataset (valid, real{\_}wrong{\_}content, fabricated) derived from CourtListener opinions via a hybrid extraction and dual-model validation pipeline, and evaluate two model families (Llama, GPT-OSS) plus a cross-family adversarial pairing across control, authority-appeal, and fabricated-citation conditions, both ungrounded and retrieval-grounded. Within-model logistic regressions with item-clustered robust standard errors show that authority appeals massively increase the odds of a correct GPT-OSS 20B verification flipping to incorrect (OR=23.333, p{\ensuremath{<}}0.0001) and show strong effects in Llama 70B (OR=11.953, p=0.0029), avoiding small-model contamination. Retrieval grounding broadly reduces flip rates in large models (GPT-OSS 120B OR{\_}Grounded=0.123; Llama 70B OR{\_}Grounded=0.239). Cross-family attacks (GPT-OSS 120B {\textrightarrow} Llama 70B) achieve 98.0{\%} ungrounded authority true{\_}ASR vs. a 70.6{\%} control baseline, and grounding does not eliminate this gap (+39.7pp, p{\ensuremath{<}}0.000001). Beyond these findings, we argue that persuasion benchmarks require a content-free control condition and a Confidence-Weighted Attack Success Rate (CW-ASR) to separate genuine ``confident override'' from ``hesitant doubt'': authority appeals make verifiers confidently wrong (mean final confidence 8.56/10; 96.8{\%} of flips at confidence {\ensuremath{\geq}} 8), whereas content-free reconsideration produces hesitant, low-confidence reversals (mean 6.80/10)."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="shah-etal-2026-persuading">
<titleInfo>
<title>Persuading the Judge: Adversarial Robustness and RAG Mitigation in LLM-Based Legal Citation Verification</title>
</titleInfo>
<name type="personal">
<namePart type="given">Rohann</namePart>
<namePart type="given">Rahul</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rishabh</namePart>
<namePart type="given">Kartik</namePart>
<namePart type="family">Jawagal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gowri</namePart>
<namePart type="family">Srinivasa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Natural Legal Language Processing Workshop 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nikolaos</namePart>
<namePart type="family">Aletras</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ilias</namePart>
<namePart type="family">Chalkidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leslie</namePart>
<namePart type="family">Barrett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cătălina</namePart>
<namePart type="family">Goanță</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Preoțiuc-Pietro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gerasimos</namePart>
<namePart type="family">Spanakis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shanshan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-494-1</identifier>
</relatedItem>
<abstract>Automated legal citation verification systems are critical for ensuring the factual integrity of AI-assisted legal research and drafting, but large language models (LLMs) serving as verifiers remain vulnerable to adversarial persuasion. We introduce a three-class benchmark dataset (valid, real_wrong_content, fabricated) derived from CourtListener opinions via a hybrid extraction and dual-model validation pipeline, and evaluate two model families (Llama, GPT-OSS) plus a cross-family adversarial pairing across control, authority-appeal, and fabricated-citation conditions, both ungrounded and retrieval-grounded. Within-model logistic regressions with item-clustered robust standard errors show that authority appeals massively increase the odds of a correct GPT-OSS 20B verification flipping to incorrect (OR=23.333, p\ensuremath<0.0001) and show strong effects in Llama 70B (OR=11.953, p=0.0029), avoiding small-model contamination. Retrieval grounding broadly reduces flip rates in large models (GPT-OSS 120B OR_Grounded=0.123; Llama 70B OR_Grounded=0.239). Cross-family attacks (GPT-OSS 120B → Llama 70B) achieve 98.0% ungrounded authority true_ASR vs. a 70.6% control baseline, and grounding does not eliminate this gap (+39.7pp, p\ensuremath<0.000001). Beyond these findings, we argue that persuasion benchmarks require a content-free control condition and a Confidence-Weighted Attack Success Rate (CW-ASR) to separate genuine “confident override” from “hesitant doubt”: authority appeals make verifiers confidently wrong (mean final confidence 8.56/10; 96.8% of flips at confidence \ensuremath\geq 8), whereas content-free reconsideration produces hesitant, low-confidence reversals (mean 6.80/10).</abstract>
<identifier type="citekey">shah-etal-2026-persuading</identifier>
<location>
<url>https://aclanthology.org/2026.nllp-1.6/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>42</start>
<end>54</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Persuading the Judge: Adversarial Robustness and RAG Mitigation in LLM-Based Legal Citation Verification
%A Shah, Rohann Rahul
%A Jawagal, Rishabh Kartik
%A Srinivasa, Gowri
%Y Aletras, Nikolaos
%Y Chalkidis, Ilias
%Y Barrett, Leslie
%Y Goanță, Cătălina
%Y Preoțiuc-Pietro, Daniel
%Y Spanakis, Gerasimos
%Y Xu, Shanshan
%S Proceedings of the Natural Legal Language Processing Workshop 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-494-1
%F shah-etal-2026-persuading
%X Automated legal citation verification systems are critical for ensuring the factual integrity of AI-assisted legal research and drafting, but large language models (LLMs) serving as verifiers remain vulnerable to adversarial persuasion. We introduce a three-class benchmark dataset (valid, real_wrong_content, fabricated) derived from CourtListener opinions via a hybrid extraction and dual-model validation pipeline, and evaluate two model families (Llama, GPT-OSS) plus a cross-family adversarial pairing across control, authority-appeal, and fabricated-citation conditions, both ungrounded and retrieval-grounded. Within-model logistic regressions with item-clustered robust standard errors show that authority appeals massively increase the odds of a correct GPT-OSS 20B verification flipping to incorrect (OR=23.333, p\ensuremath<0.0001) and show strong effects in Llama 70B (OR=11.953, p=0.0029), avoiding small-model contamination. Retrieval grounding broadly reduces flip rates in large models (GPT-OSS 120B OR_Grounded=0.123; Llama 70B OR_Grounded=0.239). Cross-family attacks (GPT-OSS 120B → Llama 70B) achieve 98.0% ungrounded authority true_ASR vs. a 70.6% control baseline, and grounding does not eliminate this gap (+39.7pp, p\ensuremath<0.000001). Beyond these findings, we argue that persuasion benchmarks require a content-free control condition and a Confidence-Weighted Attack Success Rate (CW-ASR) to separate genuine “confident override” from “hesitant doubt”: authority appeals make verifiers confidently wrong (mean final confidence 8.56/10; 96.8% of flips at confidence \ensuremath\geq 8), whereas content-free reconsideration produces hesitant, low-confidence reversals (mean 6.80/10).
%U https://aclanthology.org/2026.nllp-1.6/
%P 42-54
Markdown (Informal)
[Persuading the Judge: Adversarial Robustness and RAG Mitigation in LLM-Based Legal Citation Verification](https://aclanthology.org/2026.nllp-1.6/) (Shah et al., NLLP 2026)
ACL