@inproceedings{muscato-etal-2026-disagreeing,
title = "Disagreeing Rationales: Rethinking Classification and Explainability Evaluation in Hate Speech Detection",
author = "Muscato, Benedetta and
Chen, Beiduo and
Gezici, Gizem and
Peng, Siyao and
Plank, Barbara and
Giannotti, Fosca",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.266/",
pages = "5586--5602",
ISBN = "979-8-89176-462-0",
abstract = "Human disagreement is ubiquitous and well-known in labeling. However, variation in explanations, captured through token-level human rationales, remains far less explored. At the same time, it is unclear how to best evaluate human labels and rationales, or how to best aggregate rationales beyond majority vote, in light of this variation. Yet rationales can reveal the richness of human reasoning, which may differ in style, values, and interpretations, especially in subjective NLP tasks such as hate-speech detection. In this work, we unify diverse models, training strategies, loss functions, and existing evaluation metrics under a single protocol by systematically re-implementing them across different label and rationale representation spaces. Classification metrics are organized around two key properties{---}predictive and distributional{---}while explainability metrics through three complementary dimensions: plausibility, faithfulness, and complexity. In this unified evaluation framework, we assess model behavior across classification and explainability metrics, as well as metric sensitivity to the choice of label (hard and soft) and rationale representation space (hard, intermediate and soft). Results show that intermediate and soft representations better capture variation in labels and rationales, with metrics revealing complementary facets of the same property. These findings highlight the need to align evaluation with task goals and rethink subjective NLP evaluation to better reflect diverse human judgments."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="muscato-etal-2026-disagreeing">
<titleInfo>
<title>Disagreeing Rationales: Rethinking Classification and Explainability Evaluation in Hate Speech Detection</title>
</titleInfo>
<name type="personal">
<namePart type="given">Benedetta</namePart>
<namePart type="family">Muscato</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Beiduo</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gizem</namePart>
<namePart type="family">Gezici</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siyao</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fosca</namePart>
<namePart type="family">Giannotti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Human disagreement is ubiquitous and well-known in labeling. However, variation in explanations, captured through token-level human rationales, remains far less explored. At the same time, it is unclear how to best evaluate human labels and rationales, or how to best aggregate rationales beyond majority vote, in light of this variation. Yet rationales can reveal the richness of human reasoning, which may differ in style, values, and interpretations, especially in subjective NLP tasks such as hate-speech detection. In this work, we unify diverse models, training strategies, loss functions, and existing evaluation metrics under a single protocol by systematically re-implementing them across different label and rationale representation spaces. Classification metrics are organized around two key properties—predictive and distributional—while explainability metrics through three complementary dimensions: plausibility, faithfulness, and complexity. In this unified evaluation framework, we assess model behavior across classification and explainability metrics, as well as metric sensitivity to the choice of label (hard and soft) and rationale representation space (hard, intermediate and soft). Results show that intermediate and soft representations better capture variation in labels and rationales, with metrics revealing complementary facets of the same property. These findings highlight the need to align evaluation with task goals and rethink subjective NLP evaluation to better reflect diverse human judgments.</abstract>
<identifier type="citekey">muscato-etal-2026-disagreeing</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.266/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>5586</start>
<end>5602</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Disagreeing Rationales: Rethinking Classification and Explainability Evaluation in Hate Speech Detection
%A Muscato, Benedetta
%A Chen, Beiduo
%A Gezici, Gizem
%A Peng, Siyao
%A Plank, Barbara
%A Giannotti, Fosca
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F muscato-etal-2026-disagreeing
%X Human disagreement is ubiquitous and well-known in labeling. However, variation in explanations, captured through token-level human rationales, remains far less explored. At the same time, it is unclear how to best evaluate human labels and rationales, or how to best aggregate rationales beyond majority vote, in light of this variation. Yet rationales can reveal the richness of human reasoning, which may differ in style, values, and interpretations, especially in subjective NLP tasks such as hate-speech detection. In this work, we unify diverse models, training strategies, loss functions, and existing evaluation metrics under a single protocol by systematically re-implementing them across different label and rationale representation spaces. Classification metrics are organized around two key properties—predictive and distributional—while explainability metrics through three complementary dimensions: plausibility, faithfulness, and complexity. In this unified evaluation framework, we assess model behavior across classification and explainability metrics, as well as metric sensitivity to the choice of label (hard and soft) and rationale representation space (hard, intermediate and soft). Results show that intermediate and soft representations better capture variation in labels and rationales, with metrics revealing complementary facets of the same property. These findings highlight the need to align evaluation with task goals and rethink subjective NLP evaluation to better reflect diverse human judgments.
%U https://aclanthology.org/2026.findings-emnlp.266/
%P 5586-5602
Markdown (Informal)
[Disagreeing Rationales: Rethinking Classification and Explainability Evaluation in Hate Speech Detection](https://aclanthology.org/2026.findings-emnlp.266/) (Muscato et al., Findings 2026)
ACL