@inproceedings{liang-etal-2026-dvd,
title = "{DVD}: A Robust Method for Detecting Variant Contamination in Large Language Model Evaluation",
author = "Liang, Renzhao and
Chen, Jingru and
Jia, Bo and
Deng, Bo and
Xie, Chenggang and
Wang, Yidong and
Jin, Ke and
Wang, Xin and
Zhang, Linfeng and
Wang, Cunxiang",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1289/",
pages = "29366--29384",
ISBN = "979-8-89176-459-0",
abstract = "Large language model (LLM) evaluation is increasingly confounded by *variant contamination*: training corpora contain semantically equivalent but lexically or syntactically altered test items. These variants can evade detectors based on sampling consistency or perplexity, inflating benchmark scores through memorization rather than reasoning. We formalize this problem and introduce **DVD** (**D**etection via **V**ariance of generation **D**istribution), a detector requiring only the test item, without external reference corpora or paired contaminated/clean sets. DVD models the local output distribution through repeated temperature sampling. Our key insight is that contaminated items alternate between *memory-adherence* and *perturbation-drift* states, producing unusually high variance in the synthetic difficulty of low-probability tokens, whereas uncontaminated items remain in drift with comparatively stable difficulty values. We construct the first dedicated variant-contamination benchmark across Omni-MATH and SuperGPQA by generating and filtering semantically equivalent variants, and simulate contamination by fine-tuning Qwen2.5 and Llama3.1 models of different scales. In most evaluated settings across datasets, model scales, architectures, and training configurations, **DVD** outperforms perplexity-based, Min-*k*{\%}++, edit-distance (CDD), and embedding-similarity baselines, while showing strong robustness to hyperparameters. These results support generation-distribution variance as a principled and practical signal for detecting variant contamination, helping improve the reliability of LLM evaluation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liang-etal-2026-dvd">
<titleInfo>
<title>DVD: A Robust Method for Detecting Variant Contamination in Large Language Model Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Renzhao</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingru</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bo</namePart>
<namePart type="family">Jia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bo</namePart>
<namePart type="family">Deng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chenggang</namePart>
<namePart type="family">Xie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yidong</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ke</namePart>
<namePart type="family">Jin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Linfeng</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cunxiang</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large language model (LLM) evaluation is increasingly confounded by *variant contamination*: training corpora contain semantically equivalent but lexically or syntactically altered test items. These variants can evade detectors based on sampling consistency or perplexity, inflating benchmark scores through memorization rather than reasoning. We formalize this problem and introduce **DVD** (**D**etection via **V**ariance of generation **D**istribution), a detector requiring only the test item, without external reference corpora or paired contaminated/clean sets. DVD models the local output distribution through repeated temperature sampling. Our key insight is that contaminated items alternate between *memory-adherence* and *perturbation-drift* states, producing unusually high variance in the synthetic difficulty of low-probability tokens, whereas uncontaminated items remain in drift with comparatively stable difficulty values. We construct the first dedicated variant-contamination benchmark across Omni-MATH and SuperGPQA by generating and filtering semantically equivalent variants, and simulate contamination by fine-tuning Qwen2.5 and Llama3.1 models of different scales. In most evaluated settings across datasets, model scales, architectures, and training configurations, **DVD** outperforms perplexity-based, Min-*k*%++, edit-distance (CDD), and embedding-similarity baselines, while showing strong robustness to hyperparameters. These results support generation-distribution variance as a principled and practical signal for detecting variant contamination, helping improve the reliability of LLM evaluation.</abstract>
<identifier type="citekey">liang-etal-2026-dvd</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1289/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>29366</start>
<end>29384</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T DVD: A Robust Method for Detecting Variant Contamination in Large Language Model Evaluation
%A Liang, Renzhao
%A Chen, Jingru
%A Jia, Bo
%A Deng, Bo
%A Xie, Chenggang
%A Wang, Yidong
%A Jin, Ke
%A Wang, Xin
%A Zhang, Linfeng
%A Wang, Cunxiang
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F liang-etal-2026-dvd
%X Large language model (LLM) evaluation is increasingly confounded by *variant contamination*: training corpora contain semantically equivalent but lexically or syntactically altered test items. These variants can evade detectors based on sampling consistency or perplexity, inflating benchmark scores through memorization rather than reasoning. We formalize this problem and introduce **DVD** (**D**etection via **V**ariance of generation **D**istribution), a detector requiring only the test item, without external reference corpora or paired contaminated/clean sets. DVD models the local output distribution through repeated temperature sampling. Our key insight is that contaminated items alternate between *memory-adherence* and *perturbation-drift* states, producing unusually high variance in the synthetic difficulty of low-probability tokens, whereas uncontaminated items remain in drift with comparatively stable difficulty values. We construct the first dedicated variant-contamination benchmark across Omni-MATH and SuperGPQA by generating and filtering semantically equivalent variants, and simulate contamination by fine-tuning Qwen2.5 and Llama3.1 models of different scales. In most evaluated settings across datasets, model scales, architectures, and training configurations, **DVD** outperforms perplexity-based, Min-*k*%++, edit-distance (CDD), and embedding-similarity baselines, while showing strong robustness to hyperparameters. These results support generation-distribution variance as a principled and practical signal for detecting variant contamination, helping improve the reliability of LLM evaluation.
%U https://aclanthology.org/2026.emnlp-main.1289/
%P 29366-29384
Markdown (Informal)
[DVD: A Robust Method for Detecting Variant Contamination in Large Language Model Evaluation](https://aclanthology.org/2026.emnlp-main.1289/) (Liang et al., EMNLP 2026)
ACL
- Renzhao Liang, Jingru Chen, Bo Jia, Bo Deng, Chenggang Xie, Yidong Wang, Ke Jin, Xin Wang, Linfeng Zhang, and Cunxiang Wang. 2026. DVD: A Robust Method for Detecting Variant Contamination in Large Language Model Evaluation. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 29366–29384, Budapest, Hungary. Association for Computational Linguistics.