@inproceedings{liu-etal-2026-mcpagentbench,
title = "{MCPA}gent{B}ench: A Real-world Task Benchmark for Evaluating {LLM} Agent {MCP} Tool Use",
author = "Liu, Zixiang and
Liu, Wenrui and
Dai, Elsie and
Yu, Wenhan and
Yu, Lei and
Yang, Tong and
Han, Yinjun and
Gao, Hong",
editor = "Kamalloo, Ehsan and
Gontier, Nicolas and
Rozanov, Nikolai and
Hashemzadeh, Maryam and
Arora, Akhil",
booktitle = "Proceedings of the Second Workshop for {RE}search on Agent Language Models ({REALM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.realm-1.51/",
pages = "764--773",
ISBN = "979-8-89176-485-9",
abstract = "Large Language Models (LLMs) are increasingly serving as autonomous agents, and their utilization of external tools via the Model Context Protocol (MCP) is considered a future trend. Current MCP evaluation sets suffer from issues such as reliance on external MCP services and a lack of difficulty awareness. To address these limitations, we propose MCPAgentBench, a benchmark based on real-world MCP definitions designed to evaluate the tool-use capabilities of agents. We construct a dataset containing authentic tasks and simulated MCP tools. The evaluation employs a dynamic sandbox environment that presents agents with candidate tool lists containing distractors, thereby testing their tool selection and discrimination abilities. Furthermore, we introduce comprehensive metrics to measure both task completion rates and execution efficiency. Experiments conducted on state-of-the-art LLMs reveal significant performance differences in handling complex, multi-step tool invocations. All code is open-source at https://github.com/Brunestuder/MCPAgentBench."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-mcpagentbench">
<titleInfo>
<title>MCPAgentBench: A Real-world Task Benchmark for Evaluating LLM Agent MCP Tool Use</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zixiang</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenrui</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elsie</namePart>
<namePart type="family">Dai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenhan</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lei</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yinjun</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hong</namePart>
<namePart type="family">Gao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ehsan</namePart>
<namePart type="family">Kamalloo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicolas</namePart>
<namePart type="family">Gontier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikolai</namePart>
<namePart type="family">Rozanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maryam</namePart>
<namePart type="family">Hashemzadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akhil</namePart>
<namePart type="family">Arora</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-485-9</identifier>
</relatedItem>
<abstract>Large Language Models (LLMs) are increasingly serving as autonomous agents, and their utilization of external tools via the Model Context Protocol (MCP) is considered a future trend. Current MCP evaluation sets suffer from issues such as reliance on external MCP services and a lack of difficulty awareness. To address these limitations, we propose MCPAgentBench, a benchmark based on real-world MCP definitions designed to evaluate the tool-use capabilities of agents. We construct a dataset containing authentic tasks and simulated MCP tools. The evaluation employs a dynamic sandbox environment that presents agents with candidate tool lists containing distractors, thereby testing their tool selection and discrimination abilities. Furthermore, we introduce comprehensive metrics to measure both task completion rates and execution efficiency. Experiments conducted on state-of-the-art LLMs reveal significant performance differences in handling complex, multi-step tool invocations. All code is open-source at https://github.com/Brunestuder/MCPAgentBench.</abstract>
<identifier type="citekey">liu-etal-2026-mcpagentbench</identifier>
<location>
<url>https://aclanthology.org/2026.realm-1.51/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>764</start>
<end>773</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MCPAgentBench: A Real-world Task Benchmark for Evaluating LLM Agent MCP Tool Use
%A Liu, Zixiang
%A Liu, Wenrui
%A Dai, Elsie
%A Yu, Wenhan
%A Yu, Lei
%A Yang, Tong
%A Han, Yinjun
%A Gao, Hong
%Y Kamalloo, Ehsan
%Y Gontier, Nicolas
%Y Rozanov, Nikolai
%Y Hashemzadeh, Maryam
%Y Arora, Akhil
%S Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-485-9
%F liu-etal-2026-mcpagentbench
%X Large Language Models (LLMs) are increasingly serving as autonomous agents, and their utilization of external tools via the Model Context Protocol (MCP) is considered a future trend. Current MCP evaluation sets suffer from issues such as reliance on external MCP services and a lack of difficulty awareness. To address these limitations, we propose MCPAgentBench, a benchmark based on real-world MCP definitions designed to evaluate the tool-use capabilities of agents. We construct a dataset containing authentic tasks and simulated MCP tools. The evaluation employs a dynamic sandbox environment that presents agents with candidate tool lists containing distractors, thereby testing their tool selection and discrimination abilities. Furthermore, we introduce comprehensive metrics to measure both task completion rates and execution efficiency. Experiments conducted on state-of-the-art LLMs reveal significant performance differences in handling complex, multi-step tool invocations. All code is open-source at https://github.com/Brunestuder/MCPAgentBench.
%U https://aclanthology.org/2026.realm-1.51/
%P 764-773
Markdown (Informal)
[MCPAgentBench: A Real-world Task Benchmark for Evaluating LLM Agent MCP Tool Use](https://aclanthology.org/2026.realm-1.51/) (Liu et al., REALM 2026)
ACL
- Zixiang Liu, Wenrui Liu, Elsie Dai, Wenhan Yu, Lei Yu, Tong Yang, Yinjun Han, and Hong Gao. 2026. MCPAgentBench: A Real-world Task Benchmark for Evaluating LLM Agent MCP Tool Use. In Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026), pages 764–773, Budapest, Hungary. Association for Computational Linguistics.