@inproceedings{nakka-etal-2026-privacyevalclaw,
title = "{P}rivacy{E}val{C}law: A Tool-Augmented Deliberative Agent for Auditing Privacy Leakage in {LLM} Jailbreak Responses",
author = "Nakka, Krishna Kanth and
Wang, Yujun and
Zhou, Shijie and
Nguyen, Tu and
Gelardi, Gabriele and
Zhou, Xuebing",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.47/",
pages = "900--936",
ISBN = "979-8-89176-444-6",
abstract = "Privacy auditing of large language models (LLMs) increasingly relies on advanced jailbreak attacks to uncover unintended disclosures of personal information; the reliability of such audits, however, hinges on evaluators that can accurately determine whether a response constitutes privacy leakage. This evaluation remains challenging because disclosures may be indirect, contextual, partial, or obfuscated, and different evaluators can interpret the same response differently. We conduct a systematic study of guard models and LLM-as-Judges and find substantial disagreement among evaluators, particularly in ambiguous cases that require additional evidence or targeted analysis to resolve. Motivated by these findings, we introduce \textsc{PrivacyEvalClaw}, a training-free deliberative agent for evaluating privacy leakage in LLM jailbreak responses. Rather than relying on isolated single-pass judgments, PrivacyEvalClaw treats evaluators and analysis modules as tools, adaptively gathers evidence, investigates uncertain disclosures, and reconciles conflicting signals before producing a final judgment. Our primary experiments use pairwise LLM-as-Judge evaluation of complete trajectories, showing that PrivacyEvalClaw achieves favorable win rates against competing baselines across outcome, process, and overall quality, while also producing explicit reasoning traces that make the evaluation process more transparent and auditable."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="nakka-etal-2026-privacyevalclaw">
<titleInfo>
<title>PrivacyEvalClaw: A Tool-Augmented Deliberative Agent for Auditing Privacy Leakage in LLM Jailbreak Responses</title>
</titleInfo>
<name type="personal">
<namePart type="given">Krishna</namePart>
<namePart type="given">Kanth</namePart>
<namePart type="family">Nakka</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yujun</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shijie</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tu</namePart>
<namePart type="family">Nguyen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriele</namePart>
<namePart type="family">Gelardi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xuebing</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>Privacy auditing of large language models (LLMs) increasingly relies on advanced jailbreak attacks to uncover unintended disclosures of personal information; the reliability of such audits, however, hinges on evaluators that can accurately determine whether a response constitutes privacy leakage. This evaluation remains challenging because disclosures may be indirect, contextual, partial, or obfuscated, and different evaluators can interpret the same response differently. We conduct a systematic study of guard models and LLM-as-Judges and find substantial disagreement among evaluators, particularly in ambiguous cases that require additional evidence or targeted analysis to resolve. Motivated by these findings, we introduce PrivacyEvalClaw, a training-free deliberative agent for evaluating privacy leakage in LLM jailbreak responses. Rather than relying on isolated single-pass judgments, PrivacyEvalClaw treats evaluators and analysis modules as tools, adaptively gathers evidence, investigates uncertain disclosures, and reconciles conflicting signals before producing a final judgment. Our primary experiments use pairwise LLM-as-Judge evaluation of complete trajectories, showing that PrivacyEvalClaw achieves favorable win rates against competing baselines across outcome, process, and overall quality, while also producing explicit reasoning traces that make the evaluation process more transparent and auditable.</abstract>
<identifier type="citekey">nakka-etal-2026-privacyevalclaw</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.47/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>900</start>
<end>936</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T PrivacyEvalClaw: A Tool-Augmented Deliberative Agent for Auditing Privacy Leakage in LLM Jailbreak Responses
%A Nakka, Krishna Kanth
%A Wang, Yujun
%A Zhou, Shijie
%A Nguyen, Tu
%A Gelardi, Gabriele
%A Zhou, Xuebing
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F nakka-etal-2026-privacyevalclaw
%X Privacy auditing of large language models (LLMs) increasingly relies on advanced jailbreak attacks to uncover unintended disclosures of personal information; the reliability of such audits, however, hinges on evaluators that can accurately determine whether a response constitutes privacy leakage. This evaluation remains challenging because disclosures may be indirect, contextual, partial, or obfuscated, and different evaluators can interpret the same response differently. We conduct a systematic study of guard models and LLM-as-Judges and find substantial disagreement among evaluators, particularly in ambiguous cases that require additional evidence or targeted analysis to resolve. Motivated by these findings, we introduce PrivacyEvalClaw, a training-free deliberative agent for evaluating privacy leakage in LLM jailbreak responses. Rather than relying on isolated single-pass judgments, PrivacyEvalClaw treats evaluators and analysis modules as tools, adaptively gathers evidence, investigates uncertain disclosures, and reconciles conflicting signals before producing a final judgment. Our primary experiments use pairwise LLM-as-Judge evaluation of complete trajectories, showing that PrivacyEvalClaw achieves favorable win rates against competing baselines across outcome, process, and overall quality, while also producing explicit reasoning traces that make the evaluation process more transparent and auditable.
%U https://aclanthology.org/2026.findings-aacl.47/
%P 900-936
Markdown (Informal)
[PrivacyEvalClaw: A Tool-Augmented Deliberative Agent for Auditing Privacy Leakage in LLM Jailbreak Responses](https://aclanthology.org/2026.findings-aacl.47/) (Nakka et al., Findings 2026)
ACL