@inproceedings{miyaji-2026-mace,
title = "{MACE}-{XAI}: A Multi-Agent Crowd Evaluation Framework for Robust Natural Language Explanations",
author = "Miyaji, Renato Okabayashi",
editor = "Arad, Dana and
Belinkov, Yonatan and
Chen, Hanjie and
Feldhus, Nils and
Mueller, Aaron and
Mohebbi, Hosein and
Sarti, Gabriele and
Tutek, Martin",
booktitle = "Proceedings of the 9th {B}lackbox{NLP} Workshop: Analyzing and Interpreting Neural Networks for {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.blackboxnlp-1.50/",
pages = "879--891",
ISBN = "979-8-89176-476-7",
abstract = "The rapid integration of Machine Learning in critical decision-making has shifted the focus of Explainable Artificial Intelligence (XAI) toward human-centric Natural Language Explanations (NLEs). While Large Language Models (LLMs) can synthesize narrative justifications from technical feature attributions, ensuring the faithfulness and robustness of these explanations remains an open challenge. Although multi-agent ensembles and Crowd-based Comparative Evaluation (CCE) are emerging standards for evaluating open-ended dialogues, they fail to capture domain-specific hallucinations and structural omissions inherent to technical explanations. To bridge this gap, we introduce MACE-XAI, a novel neuro-symbolic framework engineered specifically for the robust assessment of NLEs in the context of tabular feature-attribution XAI. Rather than proposing isolated evaluators, our key innovation lies in the architectural synergy of its components: we orchestrate a heterogeneous crowd of LLM agents to evaluate pragmatic text quality and fluency, while anchoring them with a deterministic symbolic verification layer that audits textual claims against raw technical ground truths. Experimental results demonstrate that while standalone LLM-as-judge setups are easily swayed by linguistic surface features, MACE-XAI achieves an agreement with human experts (Cohen{'}s $\kappa=0.64$). While this represents a significant improvement over standalone neural judges, it also highlights the remaining alignment gap that must be addressed before fully autonomous evaluation in safety-critical deployments."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="miyaji-2026-mace">
<titleInfo>
<title>MACE-XAI: A Multi-Agent Crowd Evaluation Framework for Robust Natural Language Explanations</title>
</titleInfo>
<name type="personal">
<namePart type="given">Renato</namePart>
<namePart type="given">Okabayashi</namePart>
<namePart type="family">Miyaji</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dana</namePart>
<namePart type="family">Arad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yonatan</namePart>
<namePart type="family">Belinkov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hanjie</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nils</namePart>
<namePart type="family">Feldhus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hosein</namePart>
<namePart type="family">Mohebbi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriele</namePart>
<namePart type="family">Sarti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Tutek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-476-7</identifier>
</relatedItem>
<abstract>The rapid integration of Machine Learning in critical decision-making has shifted the focus of Explainable Artificial Intelligence (XAI) toward human-centric Natural Language Explanations (NLEs). While Large Language Models (LLMs) can synthesize narrative justifications from technical feature attributions, ensuring the faithfulness and robustness of these explanations remains an open challenge. Although multi-agent ensembles and Crowd-based Comparative Evaluation (CCE) are emerging standards for evaluating open-ended dialogues, they fail to capture domain-specific hallucinations and structural omissions inherent to technical explanations. To bridge this gap, we introduce MACE-XAI, a novel neuro-symbolic framework engineered specifically for the robust assessment of NLEs in the context of tabular feature-attribution XAI. Rather than proposing isolated evaluators, our key innovation lies in the architectural synergy of its components: we orchestrate a heterogeneous crowd of LLM agents to evaluate pragmatic text quality and fluency, while anchoring them with a deterministic symbolic verification layer that audits textual claims against raw technical ground truths. Experimental results demonstrate that while standalone LLM-as-judge setups are easily swayed by linguistic surface features, MACE-XAI achieves an agreement with human experts (Cohen’s ąppa=0.64). While this represents a significant improvement over standalone neural judges, it also highlights the remaining alignment gap that must be addressed before fully autonomous evaluation in safety-critical deployments.</abstract>
<identifier type="citekey">miyaji-2026-mace</identifier>
<location>
<url>https://aclanthology.org/2026.blackboxnlp-1.50/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>879</start>
<end>891</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MACE-XAI: A Multi-Agent Crowd Evaluation Framework for Robust Natural Language Explanations
%A Miyaji, Renato Okabayashi
%Y Arad, Dana
%Y Belinkov, Yonatan
%Y Chen, Hanjie
%Y Feldhus, Nils
%Y Mueller, Aaron
%Y Mohebbi, Hosein
%Y Sarti, Gabriele
%Y Tutek, Martin
%S Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-476-7
%F miyaji-2026-mace
%X The rapid integration of Machine Learning in critical decision-making has shifted the focus of Explainable Artificial Intelligence (XAI) toward human-centric Natural Language Explanations (NLEs). While Large Language Models (LLMs) can synthesize narrative justifications from technical feature attributions, ensuring the faithfulness and robustness of these explanations remains an open challenge. Although multi-agent ensembles and Crowd-based Comparative Evaluation (CCE) are emerging standards for evaluating open-ended dialogues, they fail to capture domain-specific hallucinations and structural omissions inherent to technical explanations. To bridge this gap, we introduce MACE-XAI, a novel neuro-symbolic framework engineered specifically for the robust assessment of NLEs in the context of tabular feature-attribution XAI. Rather than proposing isolated evaluators, our key innovation lies in the architectural synergy of its components: we orchestrate a heterogeneous crowd of LLM agents to evaluate pragmatic text quality and fluency, while anchoring them with a deterministic symbolic verification layer that audits textual claims against raw technical ground truths. Experimental results demonstrate that while standalone LLM-as-judge setups are easily swayed by linguistic surface features, MACE-XAI achieves an agreement with human experts (Cohen’s ąppa=0.64). While this represents a significant improvement over standalone neural judges, it also highlights the remaining alignment gap that must be addressed before fully autonomous evaluation in safety-critical deployments.
%U https://aclanthology.org/2026.blackboxnlp-1.50/
%P 879-891
Markdown (Informal)
[MACE-XAI: A Multi-Agent Crowd Evaluation Framework for Robust Natural Language Explanations](https://aclanthology.org/2026.blackboxnlp-1.50/) (Miyaji, BlackboxNLP 2026)
ACL