@inproceedings{baral-sigdel-2026-toolmisusebench,
title = "{T}ool{M}isuse{B}ench: An Offline Deterministic Benchmark for Tool Misuse and Recovery in Agentic Systems",
author = "Baral, Rista and
Sigdel, Akshey",
editor = "Kamalloo, Ehsan and
Gontier, Nicolas and
Rozanov, Nikolai and
Hashemzadeh, Maryam and
Arora, Akhil",
booktitle = "Proceedings of the Second Workshop for {RE}search on Agent Language Models ({REALM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.realm-1.28/",
pages = "435--442",
ISBN = "979-8-89176-485-9",
abstract = "Reliable tool use is a fundamental capability for agentic systems, yet existing evaluations primarily focus on successful tool execution and provide limited insight into how agents recover from tool failures. To address this gap, we introduce ToolMisuseBench, an offline deterministic benchmark for evaluating tool misuse and fault-conditioned recovery. ToolMisuseBench evaluates recovery behavior across domains, fault types, and difficulty levels, using output-grounded success criteria and typed recoverability metadata. The seed-42 release includes 6,000 training, 900 development, and 1,200 public-test tasks covering CRUD, retrieval, file operations, and scheduling, including failures such as timeouts, rate limits, hidden first-call schema perturbations, authorization errors, and adversarial responses. Using the benchmark, we evaluate deterministic reference agents together with local 3-4B tool-calling LLMs (Qwen3-4B, Ministral-3B, and Llama-3.2-3B). On the public-test split, Parser+Repair achieves 0.767 task success and 0.736 conditional recovery success, compared with 0.158 task success when recovery is disabled. On a shared stratified 216-task diagnostic panel, Qwen3-4B, Ministral-3B, and Llama-3.2-3B achieve task success rates of 0.796, 0.903, and 0.773, respectively. Fault-level analysis reveals distinct recovery weaknesses that overall task success alone fails to capture. ToolMisuseBench provides a controlled and reproducible framework for diagnosing tool-use recovery under frozen fault plans, enabling more rigorous evaluation of robust tool-using agents beyond execution accuracy."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="baral-sigdel-2026-toolmisusebench">
<titleInfo>
<title>ToolMisuseBench: An Offline Deterministic Benchmark for Tool Misuse and Recovery in Agentic Systems</title>
</titleInfo>
<name type="personal">
<namePart type="given">Rista</namePart>
<namePart type="family">Baral</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akshey</namePart>
<namePart type="family">Sigdel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ehsan</namePart>
<namePart type="family">Kamalloo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicolas</namePart>
<namePart type="family">Gontier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikolai</namePart>
<namePart type="family">Rozanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maryam</namePart>
<namePart type="family">Hashemzadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akhil</namePart>
<namePart type="family">Arora</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-485-9</identifier>
</relatedItem>
<abstract>Reliable tool use is a fundamental capability for agentic systems, yet existing evaluations primarily focus on successful tool execution and provide limited insight into how agents recover from tool failures. To address this gap, we introduce ToolMisuseBench, an offline deterministic benchmark for evaluating tool misuse and fault-conditioned recovery. ToolMisuseBench evaluates recovery behavior across domains, fault types, and difficulty levels, using output-grounded success criteria and typed recoverability metadata. The seed-42 release includes 6,000 training, 900 development, and 1,200 public-test tasks covering CRUD, retrieval, file operations, and scheduling, including failures such as timeouts, rate limits, hidden first-call schema perturbations, authorization errors, and adversarial responses. Using the benchmark, we evaluate deterministic reference agents together with local 3-4B tool-calling LLMs (Qwen3-4B, Ministral-3B, and Llama-3.2-3B). On the public-test split, Parser+Repair achieves 0.767 task success and 0.736 conditional recovery success, compared with 0.158 task success when recovery is disabled. On a shared stratified 216-task diagnostic panel, Qwen3-4B, Ministral-3B, and Llama-3.2-3B achieve task success rates of 0.796, 0.903, and 0.773, respectively. Fault-level analysis reveals distinct recovery weaknesses that overall task success alone fails to capture. ToolMisuseBench provides a controlled and reproducible framework for diagnosing tool-use recovery under frozen fault plans, enabling more rigorous evaluation of robust tool-using agents beyond execution accuracy.</abstract>
<identifier type="citekey">baral-sigdel-2026-toolmisusebench</identifier>
<location>
<url>https://aclanthology.org/2026.realm-1.28/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>435</start>
<end>442</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T ToolMisuseBench: An Offline Deterministic Benchmark for Tool Misuse and Recovery in Agentic Systems
%A Baral, Rista
%A Sigdel, Akshey
%Y Kamalloo, Ehsan
%Y Gontier, Nicolas
%Y Rozanov, Nikolai
%Y Hashemzadeh, Maryam
%Y Arora, Akhil
%S Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-485-9
%F baral-sigdel-2026-toolmisusebench
%X Reliable tool use is a fundamental capability for agentic systems, yet existing evaluations primarily focus on successful tool execution and provide limited insight into how agents recover from tool failures. To address this gap, we introduce ToolMisuseBench, an offline deterministic benchmark for evaluating tool misuse and fault-conditioned recovery. ToolMisuseBench evaluates recovery behavior across domains, fault types, and difficulty levels, using output-grounded success criteria and typed recoverability metadata. The seed-42 release includes 6,000 training, 900 development, and 1,200 public-test tasks covering CRUD, retrieval, file operations, and scheduling, including failures such as timeouts, rate limits, hidden first-call schema perturbations, authorization errors, and adversarial responses. Using the benchmark, we evaluate deterministic reference agents together with local 3-4B tool-calling LLMs (Qwen3-4B, Ministral-3B, and Llama-3.2-3B). On the public-test split, Parser+Repair achieves 0.767 task success and 0.736 conditional recovery success, compared with 0.158 task success when recovery is disabled. On a shared stratified 216-task diagnostic panel, Qwen3-4B, Ministral-3B, and Llama-3.2-3B achieve task success rates of 0.796, 0.903, and 0.773, respectively. Fault-level analysis reveals distinct recovery weaknesses that overall task success alone fails to capture. ToolMisuseBench provides a controlled and reproducible framework for diagnosing tool-use recovery under frozen fault plans, enabling more rigorous evaluation of robust tool-using agents beyond execution accuracy.
%U https://aclanthology.org/2026.realm-1.28/
%P 435-442
Markdown (Informal)
[ToolMisuseBench: An Offline Deterministic Benchmark for Tool Misuse and Recovery in Agentic Systems](https://aclanthology.org/2026.realm-1.28/) (Baral & Sigdel, REALM 2026)
ACL