@inproceedings{hassan-etal-2026-private,
title = "Private Evaluation Protocol: Contamination-Resistant Black-Box Evaluation via Public-Slice Surrogates",
author = "Hassan, Md. Mahadi and
Rahaman, Sazzadur and
Uddin, Yusuf Sarwar and
Karmaker, Santu",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.168/",
pages = "3726--3749",
ISBN = "979-8-89176-439-2",
abstract = "Existing paradigms for black-box LLM evaluation by independent third parties share a structural weakness: the testing benchmark must reach the model being evaluated, enabling model developers to game future evaluations through logging or retraining. Ideally, rigorous evaluation would grant evaluators direct access to the model so testing can occur without exposing benchmark contents. However, proprietary black-box models cannot be released to external evaluators, creating a fundamental barrier to independent evaluation. To address this gap, we propose the Private Evaluation Protocol (PEP), a surrogate-based framework for independent evaluation of black-box models without sending the private benchmark to the model developer. PEP releases a small public slice of a held-out benchmark, queries the target model only on that slice, and trains a surrogate from the resulting responses, balancing fidelity to the target model against leakage of the private remainder over a tunable revelation budget. Across six benchmarks, composite leakage scores reliably predict behavioral exposure (r=0.788), and fine-tuned surrogates reach 0.85{--}0.90 agreement on template-heavy datasets while plateauing below 0.73 on reasoning-heavy datasets. The optimal revelation budget is dataset-dependent, and PEP estimates black-box performance with 5.88 percentage-point MAE, showing the feasibility of private surrogate-based evaluation for structured tasks while protecting the privacy of the held-out benchmark."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hassan-etal-2026-private">
<titleInfo>
<title>Private Evaluation Protocol: Contamination-Resistant Black-Box Evaluation via Public-Slice Surrogates</title>
</titleInfo>
<name type="personal">
<namePart type="given">Md.</namePart>
<namePart type="given">Mahadi</namePart>
<namePart type="family">Hassan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sazzadur</namePart>
<namePart type="family">Rahaman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yusuf</namePart>
<namePart type="given">Sarwar</namePart>
<namePart type="family">Uddin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Santu</namePart>
<namePart type="family">Karmaker</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Existing paradigms for black-box LLM evaluation by independent third parties share a structural weakness: the testing benchmark must reach the model being evaluated, enabling model developers to game future evaluations through logging or retraining. Ideally, rigorous evaluation would grant evaluators direct access to the model so testing can occur without exposing benchmark contents. However, proprietary black-box models cannot be released to external evaluators, creating a fundamental barrier to independent evaluation. To address this gap, we propose the Private Evaluation Protocol (PEP), a surrogate-based framework for independent evaluation of black-box models without sending the private benchmark to the model developer. PEP releases a small public slice of a held-out benchmark, queries the target model only on that slice, and trains a surrogate from the resulting responses, balancing fidelity to the target model against leakage of the private remainder over a tunable revelation budget. Across six benchmarks, composite leakage scores reliably predict behavioral exposure (r=0.788), and fine-tuned surrogates reach 0.85–0.90 agreement on template-heavy datasets while plateauing below 0.73 on reasoning-heavy datasets. The optimal revelation budget is dataset-dependent, and PEP estimates black-box performance with 5.88 percentage-point MAE, showing the feasibility of private surrogate-based evaluation for structured tasks while protecting the privacy of the held-out benchmark.</abstract>
<identifier type="citekey">hassan-etal-2026-private</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.168/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>3726</start>
<end>3749</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Private Evaluation Protocol: Contamination-Resistant Black-Box Evaluation via Public-Slice Surrogates
%A Hassan, Md. Mahadi
%A Rahaman, Sazzadur
%A Uddin, Yusuf Sarwar
%A Karmaker, Santu
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F hassan-etal-2026-private
%X Existing paradigms for black-box LLM evaluation by independent third parties share a structural weakness: the testing benchmark must reach the model being evaluated, enabling model developers to game future evaluations through logging or retraining. Ideally, rigorous evaluation would grant evaluators direct access to the model so testing can occur without exposing benchmark contents. However, proprietary black-box models cannot be released to external evaluators, creating a fundamental barrier to independent evaluation. To address this gap, we propose the Private Evaluation Protocol (PEP), a surrogate-based framework for independent evaluation of black-box models without sending the private benchmark to the model developer. PEP releases a small public slice of a held-out benchmark, queries the target model only on that slice, and trains a surrogate from the resulting responses, balancing fidelity to the target model against leakage of the private remainder over a tunable revelation budget. Across six benchmarks, composite leakage scores reliably predict behavioral exposure (r=0.788), and fine-tuned surrogates reach 0.85–0.90 agreement on template-heavy datasets while plateauing below 0.73 on reasoning-heavy datasets. The optimal revelation budget is dataset-dependent, and PEP estimates black-box performance with 5.88 percentage-point MAE, showing the feasibility of private surrogate-based evaluation for structured tasks while protecting the privacy of the held-out benchmark.
%U https://aclanthology.org/2026.aacl-long.168/
%P 3726-3749
Markdown (Informal)
[Private Evaluation Protocol: Contamination-Resistant Black-Box Evaluation via Public-Slice Surrogates](https://aclanthology.org/2026.aacl-long.168/) (Hassan et al., AACL-IJCNLP 2026)
ACL