@inproceedings{kopcan-etal-2026-xai,
title = "Are {XAI} Metrics Meaningful Proxies for Human Judgment? Evaluation Explanation Quality in Claim Matching",
author = "Kopcan, Jaroslav and
Kri{\v{s}}, {\v{L}}ubo{\v{s}} and
Peng, Qiwei and
Blstak, Miroslav and
Schmitt, Vera",
editor = "Arad, Dana and
Belinkov, Yonatan and
Chen, Hanjie and
Feldhus, Nils and
Mueller, Aaron and
Mohebbi, Hosein and
Sarti, Gabriele and
Tutek, Martin",
booktitle = "Proceedings of the 9th {B}lackbox{NLP} Workshop: Analyzing and Interpreting Neural Networks for {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.blackboxnlp-1.39/",
pages = "677--695",
ISBN = "979-8-89176-476-7",
abstract = "AutoXAI frameworks automatically select and optimize explainability methods using quantitative metrics such as faithfulness and plausibility. But do these metrics align with human perception of explanation quality? We investigate this question for attribution-based explanation methods, which assign token-level relevance scores to the input, in the context of claim matching, a core fact-checking task in which a bi-encoder model matches social media posts to previously fact-checked claims. Combining a large-scale evaluation of 11 XAI methods with a controlled user study, we find that methods recommended by AutoXAI metrics are perceived by users as most helpful for understanding the claim-post match, while deprioritized methods are judged as misleading. We further show that simple post-processing of explanations can improve human-perceived usefulness without degrading faithfulness. Because AutoXAI recommends the top-ranked method in practice, this alignment at the decisive top-versus-bottom boundary positions these metrics as reliable indicators for distinguishing clearly strong from clearly weak methods."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kopcan-etal-2026-xai">
<titleInfo>
<title>Are XAI Metrics Meaningful Proxies for Human Judgment? Evaluation Explanation Quality in Claim Matching</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jaroslav</namePart>
<namePart type="family">Kopcan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ľuboš</namePart>
<namePart type="family">Kriš</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qiwei</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Miroslav</namePart>
<namePart type="family">Blstak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vera</namePart>
<namePart type="family">Schmitt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dana</namePart>
<namePart type="family">Arad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yonatan</namePart>
<namePart type="family">Belinkov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hanjie</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nils</namePart>
<namePart type="family">Feldhus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hosein</namePart>
<namePart type="family">Mohebbi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriele</namePart>
<namePart type="family">Sarti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Tutek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-476-7</identifier>
</relatedItem>
<abstract>AutoXAI frameworks automatically select and optimize explainability methods using quantitative metrics such as faithfulness and plausibility. But do these metrics align with human perception of explanation quality? We investigate this question for attribution-based explanation methods, which assign token-level relevance scores to the input, in the context of claim matching, a core fact-checking task in which a bi-encoder model matches social media posts to previously fact-checked claims. Combining a large-scale evaluation of 11 XAI methods with a controlled user study, we find that methods recommended by AutoXAI metrics are perceived by users as most helpful for understanding the claim-post match, while deprioritized methods are judged as misleading. We further show that simple post-processing of explanations can improve human-perceived usefulness without degrading faithfulness. Because AutoXAI recommends the top-ranked method in practice, this alignment at the decisive top-versus-bottom boundary positions these metrics as reliable indicators for distinguishing clearly strong from clearly weak methods.</abstract>
<identifier type="citekey">kopcan-etal-2026-xai</identifier>
<location>
<url>https://aclanthology.org/2026.blackboxnlp-1.39/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>677</start>
<end>695</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Are XAI Metrics Meaningful Proxies for Human Judgment? Evaluation Explanation Quality in Claim Matching
%A Kopcan, Jaroslav
%A Kriš, Ľuboš
%A Peng, Qiwei
%A Blstak, Miroslav
%A Schmitt, Vera
%Y Arad, Dana
%Y Belinkov, Yonatan
%Y Chen, Hanjie
%Y Feldhus, Nils
%Y Mueller, Aaron
%Y Mohebbi, Hosein
%Y Sarti, Gabriele
%Y Tutek, Martin
%S Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-476-7
%F kopcan-etal-2026-xai
%X AutoXAI frameworks automatically select and optimize explainability methods using quantitative metrics such as faithfulness and plausibility. But do these metrics align with human perception of explanation quality? We investigate this question for attribution-based explanation methods, which assign token-level relevance scores to the input, in the context of claim matching, a core fact-checking task in which a bi-encoder model matches social media posts to previously fact-checked claims. Combining a large-scale evaluation of 11 XAI methods with a controlled user study, we find that methods recommended by AutoXAI metrics are perceived by users as most helpful for understanding the claim-post match, while deprioritized methods are judged as misleading. We further show that simple post-processing of explanations can improve human-perceived usefulness without degrading faithfulness. Because AutoXAI recommends the top-ranked method in practice, this alignment at the decisive top-versus-bottom boundary positions these metrics as reliable indicators for distinguishing clearly strong from clearly weak methods.
%U https://aclanthology.org/2026.blackboxnlp-1.39/
%P 677-695
Markdown (Informal)
[Are XAI Metrics Meaningful Proxies for Human Judgment? Evaluation Explanation Quality in Claim Matching](https://aclanthology.org/2026.blackboxnlp-1.39/) (Kopcan et al., BlackboxNLP 2026)
ACL