@inproceedings{li-etal-2026-macro,
title = "{MACRO}: A Benchmark for Clarification-Aware Multi-Agent Orchestration",
author = "Li, Tao and
Chen, Jianyi and
Wu, Yaqiang and
Ma, Jie and
Yang, Shangda and
Elibol, Oguz H and
Singh, Pradyumna S.",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2061/",
pages = "46914--46952",
ISBN = "979-8-89176-459-0",
abstract = "Clarification-aware multi-agent orchestration is challenging because systems must elicit missing information, maintain task state, and coordinate dependent subtasks. Existing benchmarks do not comprehensively evaluate such orchestration under underspecified user requests, focusing instead on end-to-end success or isolated abilities like tool use and dialogue. We introduce MACRO, a travel-domain benchmark designed to address these gaps by providing structured task tiers, process-level user-side and system-side milestone annotations that include expected user clarifications, and standardized sub-agent interfaces. MACRO-Full contains 107 fully annotated tasks with 703 milestones, and MACRO-Core provides a tractable 30-task subset with 164 milestones. We evaluate three representative orchestration frameworks and a reference decoupled planning{--}execution orchestrator (MACRO-DPE) using trace-level metrics and failure analysis. Results show that MACRO-DPE improves routing fidelity and execution efficiency while maintaining task completion, whereas staged planning ensures milestone coverage at higher cost. The dataset[{\ensuremath{<}}https://huggingface.co/datasets/weier248/MACRO-bench{\ensuremath{>}}] and code are publicly available at \url{https://github.com/liuweie/MACRO-bench/}."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-etal-2026-macro">
<titleInfo>
<title>MACRO: A Benchmark for Clarification-Aware Multi-Agent Orchestration</title>
</titleInfo>
<name type="personal">
<namePart type="given">Tao</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jianyi</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yaqiang</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jie</namePart>
<namePart type="family">Ma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shangda</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Oguz</namePart>
<namePart type="given">H</namePart>
<namePart type="family">Elibol</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pradyumna</namePart>
<namePart type="given">S</namePart>
<namePart type="family">Singh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Clarification-aware multi-agent orchestration is challenging because systems must elicit missing information, maintain task state, and coordinate dependent subtasks. Existing benchmarks do not comprehensively evaluate such orchestration under underspecified user requests, focusing instead on end-to-end success or isolated abilities like tool use and dialogue. We introduce MACRO, a travel-domain benchmark designed to address these gaps by providing structured task tiers, process-level user-side and system-side milestone annotations that include expected user clarifications, and standardized sub-agent interfaces. MACRO-Full contains 107 fully annotated tasks with 703 milestones, and MACRO-Core provides a tractable 30-task subset with 164 milestones. We evaluate three representative orchestration frameworks and a reference decoupled planning–execution orchestrator (MACRO-DPE) using trace-level metrics and failure analysis. Results show that MACRO-DPE improves routing fidelity and execution efficiency while maintaining task completion, whereas staged planning ensures milestone coverage at higher cost. The dataset[\ensuremath<https://huggingface.co/datasets/weier248/MACRO-bench\ensuremath>] and code are publicly available at https://github.com/liuweie/MACRO-bench/.</abstract>
<identifier type="citekey">li-etal-2026-macro</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2061/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>46914</start>
<end>46952</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MACRO: A Benchmark for Clarification-Aware Multi-Agent Orchestration
%A Li, Tao
%A Chen, Jianyi
%A Wu, Yaqiang
%A Ma, Jie
%A Yang, Shangda
%A Elibol, Oguz H.
%A Singh, Pradyumna S.
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F li-etal-2026-macro
%X Clarification-aware multi-agent orchestration is challenging because systems must elicit missing information, maintain task state, and coordinate dependent subtasks. Existing benchmarks do not comprehensively evaluate such orchestration under underspecified user requests, focusing instead on end-to-end success or isolated abilities like tool use and dialogue. We introduce MACRO, a travel-domain benchmark designed to address these gaps by providing structured task tiers, process-level user-side and system-side milestone annotations that include expected user clarifications, and standardized sub-agent interfaces. MACRO-Full contains 107 fully annotated tasks with 703 milestones, and MACRO-Core provides a tractable 30-task subset with 164 milestones. We evaluate three representative orchestration frameworks and a reference decoupled planning–execution orchestrator (MACRO-DPE) using trace-level metrics and failure analysis. Results show that MACRO-DPE improves routing fidelity and execution efficiency while maintaining task completion, whereas staged planning ensures milestone coverage at higher cost. The dataset[\ensuremath<https://huggingface.co/datasets/weier248/MACRO-bench\ensuremath>] and code are publicly available at https://github.com/liuweie/MACRO-bench/.
%U https://aclanthology.org/2026.emnlp-main.2061/
%P 46914-46952
Markdown (Informal)
[MACRO: A Benchmark for Clarification-Aware Multi-Agent Orchestration](https://aclanthology.org/2026.emnlp-main.2061/) (Li et al., EMNLP 2026)
ACL
- Tao Li, Jianyi Chen, Yaqiang Wu, Jie Ma, Shangda Yang, Oguz H Elibol, and Pradyumna S. Singh. 2026. MACRO: A Benchmark for Clarification-Aware Multi-Agent Orchestration. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 46914–46952, Budapest, Hungary. Association for Computational Linguistics.