@inproceedings{chen-etal-2026-livingarena,
title = "{L}iving{A}rena: Do {LLM}s Know What Other {LLM}s Don{'}t? Peer-Probing as Scalable Evaluation",
author = "Chen, Xingyu and
Wang, Rui and
Tu, Zhaopeng and
Bo, Liefeng",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.176/",
pages = "2984--2998",
ISBN = "979-8-89176-460-6",
abstract = "Fixed benchmarks are costly to renew and cannot adapt their questions to model-specific failures. We ask whether LLMs can instead discover one another{'}s weaknesses and turn those observations into an evaluation process. To study this question, we introduce \textbf{LivingArena}, an automated peer-probing framework in which models take turns testing one another. Using the interaction history, each questioner identifies potential weaknesses of its opponent and constructs targeted, verifiable questions to probe them. A 3,600-round tournament of ten models reveals a clear role asymmetry: strong answerers are not always reliable questioners, because they may generate internally inconsistent tests or fail to verify their own reference answers. After a questioner exposes an answerer{'}s failure, it is more likely to pursue the same capability domain, while the answerer{'}s weakness recurs on independently generated questions, including questions written by different models. These findings show that peer probing can reveal persistent model-specific weaknesses while separately evaluating answering and reliable test construction. By automating this process and allowing test difficulty to evolve with model capabilities, LivingArena provides a ``living'' benchmark for model development, red-teaming, and capability-aware multi-agent coordination. We publicly release our code.[{\ensuremath{<}}https://github.com/galaxyChen/LivingArena{\ensuremath{>}}]"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chen-etal-2026-livingarena">
<titleInfo>
<title>LivingArena: Do LLMs Know What Other LLMs Don’t? Peer-Probing as Scalable Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xingyu</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rui</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhaopeng</namePart>
<namePart type="family">Tu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Liefeng</namePart>
<namePart type="family">Bo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>Fixed benchmarks are costly to renew and cannot adapt their questions to model-specific failures. We ask whether LLMs can instead discover one another’s weaknesses and turn those observations into an evaluation process. To study this question, we introduce LivingArena, an automated peer-probing framework in which models take turns testing one another. Using the interaction history, each questioner identifies potential weaknesses of its opponent and constructs targeted, verifiable questions to probe them. A 3,600-round tournament of ten models reveals a clear role asymmetry: strong answerers are not always reliable questioners, because they may generate internally inconsistent tests or fail to verify their own reference answers. After a questioner exposes an answerer’s failure, it is more likely to pursue the same capability domain, while the answerer’s weakness recurs on independently generated questions, including questions written by different models. These findings show that peer probing can reveal persistent model-specific weaknesses while separately evaluating answering and reliable test construction. By automating this process and allowing test difficulty to evolve with model capabilities, LivingArena provides a “living” benchmark for model development, red-teaming, and capability-aware multi-agent coordination. We publicly release our code.[\ensuremath<https://github.com/galaxyChen/LivingArena\ensuremath>]</abstract>
<identifier type="citekey">chen-etal-2026-livingarena</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.176/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>2984</start>
<end>2998</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T LivingArena: Do LLMs Know What Other LLMs Don’t? Peer-Probing as Scalable Evaluation
%A Chen, Xingyu
%A Wang, Rui
%A Tu, Zhaopeng
%A Bo, Liefeng
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F chen-etal-2026-livingarena
%X Fixed benchmarks are costly to renew and cannot adapt their questions to model-specific failures. We ask whether LLMs can instead discover one another’s weaknesses and turn those observations into an evaluation process. To study this question, we introduce LivingArena, an automated peer-probing framework in which models take turns testing one another. Using the interaction history, each questioner identifies potential weaknesses of its opponent and constructs targeted, verifiable questions to probe them. A 3,600-round tournament of ten models reveals a clear role asymmetry: strong answerers are not always reliable questioners, because they may generate internally inconsistent tests or fail to verify their own reference answers. After a questioner exposes an answerer’s failure, it is more likely to pursue the same capability domain, while the answerer’s weakness recurs on independently generated questions, including questions written by different models. These findings show that peer probing can reveal persistent model-specific weaknesses while separately evaluating answering and reliable test construction. By automating this process and allowing test difficulty to evolve with model capabilities, LivingArena provides a “living” benchmark for model development, red-teaming, and capability-aware multi-agent coordination. We publicly release our code.[\ensuremath<https://github.com/galaxyChen/LivingArena\ensuremath>]
%U https://aclanthology.org/2026.emnlp-industry.176/
%P 2984-2998
Markdown (Informal)
[LivingArena: Do LLMs Know What Other LLMs Don’t? Peer-Probing as Scalable Evaluation](https://aclanthology.org/2026.emnlp-industry.176/) (Chen et al., EMNLP 2026)
ACL