@inproceedings{zhang-etal-2026-agent4weakness,
title = "{A}gent4{W}eakness: An Agentic Framework for In-Depth Model Weakness Discovery",
author = "Zhang, Xuanliang and
Yang, Chenghao and
Wen, Zhoufutu and
Wang, Dingzirui and
Zhang, Ge and
Zhao, Xiying and
Feng, Tianren and
Jiao, Jianpeng and
Liu, Jingkai and
Wang, Zaiyuan and
Wang, Zuo and
Wu, Wenya and
Huan, Zhou and
Chen, Jin and
Huang, Wenhao and
Zhu, Qingfu and
Che, Wanxiang",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1511/",
pages = "32150--32186",
ISBN = "979-8-89176-462-0",
abstract = "The rapid evolution of reasoning-intensive Large Language Models renders traditional metrics insufficient by masking fine-grained failures and implicit pathologies. Existing weakness discovery methods typically rely on rigid pipelines, yielding superficial insights that lack the diagnostic depth required for effective model improvement. To address this, we introduce Agent4Weakness, a multi-agent framework designed to replicate the rigorous workflow of human expert analysts. By integrating a Domain-Aware Memory for contextual reasoning with diagnostic evaluation skills and a Tool Abstraction mechanism for decouple data analysis, Agent4Weakness transforms raw evaluation traces into grounded, actionable reports. We validate our framework through an extensive study involving 104 models across 27 benchmarks. Experimental results demonstrate that Agent4Weakness produces diagnostic reports significantly superior to competitive baselines. Crucially, leveraging these insights for prompt guidance yields an average 3.7 point performance boost and establishes a closed-loop optimization framework."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-agent4weakness">
<titleInfo>
<title>Agent4Weakness: An Agentic Framework for In-Depth Model Weakness Discovery</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xuanliang</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chenghao</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhoufutu</namePart>
<namePart type="family">Wen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dingzirui</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ge</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiying</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianren</namePart>
<namePart type="family">Feng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jianpeng</namePart>
<namePart type="family">Jiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingkai</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zaiyuan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zuo</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenya</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhou</namePart>
<namePart type="family">Huan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jin</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenhao</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qingfu</namePart>
<namePart type="family">Zhu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wanxiang</namePart>
<namePart type="family">Che</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>The rapid evolution of reasoning-intensive Large Language Models renders traditional metrics insufficient by masking fine-grained failures and implicit pathologies. Existing weakness discovery methods typically rely on rigid pipelines, yielding superficial insights that lack the diagnostic depth required for effective model improvement. To address this, we introduce Agent4Weakness, a multi-agent framework designed to replicate the rigorous workflow of human expert analysts. By integrating a Domain-Aware Memory for contextual reasoning with diagnostic evaluation skills and a Tool Abstraction mechanism for decouple data analysis, Agent4Weakness transforms raw evaluation traces into grounded, actionable reports. We validate our framework through an extensive study involving 104 models across 27 benchmarks. Experimental results demonstrate that Agent4Weakness produces diagnostic reports significantly superior to competitive baselines. Crucially, leveraging these insights for prompt guidance yields an average 3.7 point performance boost and establishes a closed-loop optimization framework.</abstract>
<identifier type="citekey">zhang-etal-2026-agent4weakness</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1511/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>32150</start>
<end>32186</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Agent4Weakness: An Agentic Framework for In-Depth Model Weakness Discovery
%A Zhang, Xuanliang
%A Yang, Chenghao
%A Wen, Zhoufutu
%A Wang, Dingzirui
%A Zhang, Ge
%A Zhao, Xiying
%A Feng, Tianren
%A Jiao, Jianpeng
%A Liu, Jingkai
%A Wang, Zaiyuan
%A Wang, Zuo
%A Wu, Wenya
%A Huan, Zhou
%A Chen, Jin
%A Huang, Wenhao
%A Zhu, Qingfu
%A Che, Wanxiang
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F zhang-etal-2026-agent4weakness
%X The rapid evolution of reasoning-intensive Large Language Models renders traditional metrics insufficient by masking fine-grained failures and implicit pathologies. Existing weakness discovery methods typically rely on rigid pipelines, yielding superficial insights that lack the diagnostic depth required for effective model improvement. To address this, we introduce Agent4Weakness, a multi-agent framework designed to replicate the rigorous workflow of human expert analysts. By integrating a Domain-Aware Memory for contextual reasoning with diagnostic evaluation skills and a Tool Abstraction mechanism for decouple data analysis, Agent4Weakness transforms raw evaluation traces into grounded, actionable reports. We validate our framework through an extensive study involving 104 models across 27 benchmarks. Experimental results demonstrate that Agent4Weakness produces diagnostic reports significantly superior to competitive baselines. Crucially, leveraging these insights for prompt guidance yields an average 3.7 point performance boost and establishes a closed-loop optimization framework.
%U https://aclanthology.org/2026.findings-emnlp.1511/
%P 32150-32186
Markdown (Informal)
[Agent4Weakness: An Agentic Framework for In-Depth Model Weakness Discovery](https://aclanthology.org/2026.findings-emnlp.1511/) (Zhang et al., Findings 2026)
ACL
- Xuanliang Zhang, Chenghao Yang, Zhoufutu Wen, Dingzirui Wang, Ge Zhang, Xiying Zhao, Tianren Feng, Jianpeng Jiao, Jingkai Liu, Zaiyuan Wang, Zuo Wang, Wenya Wu, Zhou Huan, Jin Chen, Wenhao Huang, Qingfu Zhu, and Wanxiang Che. 2026. Agent4Weakness: An Agentic Framework for In-Depth Model Weakness Discovery. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 32150–32186, Budapest, Hungary. Association for Computational Linguistics.