@inproceedings{ke-etal-2026-dybencheval,
title = "{D}y{B}ench{E}val: A Comprehensive Evaluation Suite for Measuring {LLM}-as-Dynamic-Benchmarker",
author = "Ke, Pei and
Wang, Jingye and
Sun, Maolin and
Wang, Lan and
Peng, Haitao and
Wen, Bosi and
Hu, Enping and
Luo, Ruyi and
Huang, Minlie and
Qin, Ke",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1894/",
pages = "40342--40363",
ISBN = "979-8-89176-462-0",
abstract = "Dynamic benchmarking is an essential research direction in the evaluation of large language models (LLMs) to avoid data contamination. Although existing works have proposed LLM-based methods for dynamic benchmarking, there still lacks a unified paradigm for performance comparison among different LLMs, which hinders researchers from model selection for specific domains and tasks. To address this challenge, we propose a comprehensive evaluation suite called DyBenchEval, including a novel seed dataset across 4 domains and 7 tasks as well as multi-aspect evaluation metrics. A cross data filling method is utilized for high-quality data augmentation with domain/task-level similarity, aiming to overcome the data scarcity problem. For automatic metrics, we consider coherence, correctness, and diversity of LLM-generated evaluation samples, simultaneously measuring the respective quality of generated questions / answers and the relationship between them. Extensive experimental results on DyBenchEval reveal deficiencies in 11 mainstream closed-source and open-source LLMs when dealing with dynamic benchmarking across different domains and tasks."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ke-etal-2026-dybencheval">
<titleInfo>
<title>DyBenchEval: A Comprehensive Evaluation Suite for Measuring LLM-as-Dynamic-Benchmarker</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pei</namePart>
<namePart type="family">Ke</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingye</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maolin</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haitao</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bosi</namePart>
<namePart type="family">Wen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Enping</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruyi</namePart>
<namePart type="family">Luo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minlie</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ke</namePart>
<namePart type="family">Qin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Dynamic benchmarking is an essential research direction in the evaluation of large language models (LLMs) to avoid data contamination. Although existing works have proposed LLM-based methods for dynamic benchmarking, there still lacks a unified paradigm for performance comparison among different LLMs, which hinders researchers from model selection for specific domains and tasks. To address this challenge, we propose a comprehensive evaluation suite called DyBenchEval, including a novel seed dataset across 4 domains and 7 tasks as well as multi-aspect evaluation metrics. A cross data filling method is utilized for high-quality data augmentation with domain/task-level similarity, aiming to overcome the data scarcity problem. For automatic metrics, we consider coherence, correctness, and diversity of LLM-generated evaluation samples, simultaneously measuring the respective quality of generated questions / answers and the relationship between them. Extensive experimental results on DyBenchEval reveal deficiencies in 11 mainstream closed-source and open-source LLMs when dealing with dynamic benchmarking across different domains and tasks.</abstract>
<identifier type="citekey">ke-etal-2026-dybencheval</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1894/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>40342</start>
<end>40363</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T DyBenchEval: A Comprehensive Evaluation Suite for Measuring LLM-as-Dynamic-Benchmarker
%A Ke, Pei
%A Wang, Jingye
%A Sun, Maolin
%A Wang, Lan
%A Peng, Haitao
%A Wen, Bosi
%A Hu, Enping
%A Luo, Ruyi
%A Huang, Minlie
%A Qin, Ke
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F ke-etal-2026-dybencheval
%X Dynamic benchmarking is an essential research direction in the evaluation of large language models (LLMs) to avoid data contamination. Although existing works have proposed LLM-based methods for dynamic benchmarking, there still lacks a unified paradigm for performance comparison among different LLMs, which hinders researchers from model selection for specific domains and tasks. To address this challenge, we propose a comprehensive evaluation suite called DyBenchEval, including a novel seed dataset across 4 domains and 7 tasks as well as multi-aspect evaluation metrics. A cross data filling method is utilized for high-quality data augmentation with domain/task-level similarity, aiming to overcome the data scarcity problem. For automatic metrics, we consider coherence, correctness, and diversity of LLM-generated evaluation samples, simultaneously measuring the respective quality of generated questions / answers and the relationship between them. Extensive experimental results on DyBenchEval reveal deficiencies in 11 mainstream closed-source and open-source LLMs when dealing with dynamic benchmarking across different domains and tasks.
%U https://aclanthology.org/2026.findings-emnlp.1894/
%P 40342-40363
Markdown (Informal)
[DyBenchEval: A Comprehensive Evaluation Suite for Measuring LLM-as-Dynamic-Benchmarker](https://aclanthology.org/2026.findings-emnlp.1894/) (Ke et al., Findings 2026)
ACL
- Pei Ke, Jingye Wang, Maolin Sun, Lan Wang, Haitao Peng, Bosi Wen, Enping Hu, Ruyi Luo, Minlie Huang, and Ke Qin. 2026. DyBenchEval: A Comprehensive Evaluation Suite for Measuring LLM-as-Dynamic-Benchmarker. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 40342–40363, Budapest, Hungary. Association for Computational Linguistics.