@inproceedings{ikhwantri-marijan-2026-use,
title = "Why and When to Use Statistical Significance in Explanation Evaluation: A Systematic Survey and Guidelines for Explainable {NLP}",
author = "Ikhwantri, Fariz and
Marijan, Dusica",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.504/",
pages = "11571--11595",
ISBN = "979-8-89176-459-0",
abstract = "In this paper, we present a systematic survey and methodological guidelines for explanation evaluation and statistical significance testing in explainable NLP (XAI). We systematically reviewed 494 verified papers from the ACL Anthology (2020{--}2025) to identify how explanations are evaluated and the frequency of statistical validation. Our analysis identifies a persistent **rigour gap**: although **the number of explainable NLP papers increased fivefold** during the study period, there was **no corresponding increase in the use of explanation-specific statistical testing**. Across the full corpus, only 25.30{\%} (125/494) of studies substantiate evaluation claims with hypothesis testing. This remains a minority practice in every surveyed year. We categorise the primary barriers to rigour into three issues: implicit generalisation, inferential reporting gap, and incomplete rigour. To address these, we propose a bipartite framework that maps research hypotheses to appropriate statistical testing regimes. Our goal is to clarify when statistical testing is necessary and to provide a guideline for more rigorous, reproducible explainability research in the LLM era."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ikhwantri-marijan-2026-use">
<titleInfo>
<title>Why and When to Use Statistical Significance in Explanation Evaluation: A Systematic Survey and Guidelines for Explainable NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Fariz</namePart>
<namePart type="family">Ikhwantri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dusica</namePart>
<namePart type="family">Marijan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>In this paper, we present a systematic survey and methodological guidelines for explanation evaluation and statistical significance testing in explainable NLP (XAI). We systematically reviewed 494 verified papers from the ACL Anthology (2020–2025) to identify how explanations are evaluated and the frequency of statistical validation. Our analysis identifies a persistent **rigour gap**: although **the number of explainable NLP papers increased fivefold** during the study period, there was **no corresponding increase in the use of explanation-specific statistical testing**. Across the full corpus, only 25.30% (125/494) of studies substantiate evaluation claims with hypothesis testing. This remains a minority practice in every surveyed year. We categorise the primary barriers to rigour into three issues: implicit generalisation, inferential reporting gap, and incomplete rigour. To address these, we propose a bipartite framework that maps research hypotheses to appropriate statistical testing regimes. Our goal is to clarify when statistical testing is necessary and to provide a guideline for more rigorous, reproducible explainability research in the LLM era.</abstract>
<identifier type="citekey">ikhwantri-marijan-2026-use</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.504/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>11571</start>
<end>11595</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Why and When to Use Statistical Significance in Explanation Evaluation: A Systematic Survey and Guidelines for Explainable NLP
%A Ikhwantri, Fariz
%A Marijan, Dusica
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F ikhwantri-marijan-2026-use
%X In this paper, we present a systematic survey and methodological guidelines for explanation evaluation and statistical significance testing in explainable NLP (XAI). We systematically reviewed 494 verified papers from the ACL Anthology (2020–2025) to identify how explanations are evaluated and the frequency of statistical validation. Our analysis identifies a persistent **rigour gap**: although **the number of explainable NLP papers increased fivefold** during the study period, there was **no corresponding increase in the use of explanation-specific statistical testing**. Across the full corpus, only 25.30% (125/494) of studies substantiate evaluation claims with hypothesis testing. This remains a minority practice in every surveyed year. We categorise the primary barriers to rigour into three issues: implicit generalisation, inferential reporting gap, and incomplete rigour. To address these, we propose a bipartite framework that maps research hypotheses to appropriate statistical testing regimes. Our goal is to clarify when statistical testing is necessary and to provide a guideline for more rigorous, reproducible explainability research in the LLM era.
%U https://aclanthology.org/2026.emnlp-main.504/
%P 11571-11595
Markdown (Informal)
[Why and When to Use Statistical Significance in Explanation Evaluation: A Systematic Survey and Guidelines for Explainable NLP](https://aclanthology.org/2026.emnlp-main.504/) (Ikhwantri & Marijan, EMNLP 2026)
ACL