@inproceedings{prashanth-etal-2026-llm,
title = "Does {LLM} Instability Reflect Annotator Disagreement? A Study on Hate Speech Detection",
author = "Prashanth, Sneha and
Mahati, Aradhana and
Ziemlewski, Maksym and
Loaiza, Nicolas and
Khurana, Urja",
editor = "Calabrese, Agostina and
Davidson, Thomas and
de Kock, Christine and
Khurana, Urja and
Marchiori Manerba, Marta and
Piot, Paloma and
Talat, Zeerak",
booktitle = "Proceedings of the 10th Workshop on Online Abuse and Harms ({WOAH})",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.woah-1.38/",
pages = "466--476",
ISBN = "979-8-89176-487-3",
abstract = "Hate speech detection is inherently subjective because annotators routinely disagree on the same item, and collapsing their labels into a majority vote discards information about how contested the item actually is. Large language models exhibit variation by returning different labels for the same input across random seeds. We ask whether this variation empirically lines up with human disagreement. We operationalize annotator disagreement via entropy and the LLM instability via flip rate and model entropy across five seeds at fixed temperature, then measure their rank correlation on the full CREHate dataset using three models from both open- and closed-source families. The alignment with human disagreement is model dependent: Gemini 2.5 Flash and Llama 3.1 8B show significant positive correlations, while Mistral 7B shows only a marginal effect. The relationship is also modest in absolute scale: most items cluster at a flip rate of zero, and the signal is concentrated in a small fraction of high-disagreement items that also produce model instability. Qualitative analysis shows that culturally contested items, where human review is needed, sometimes produce zero model instability, as models classify them in opposite directions across cultural contexts. Our findings do not show evidence that seed-based instability is a reliable proxy for human disagreement for all LLMs."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="prashanth-etal-2026-llm">
<titleInfo>
<title>Does LLM Instability Reflect Annotator Disagreement? A Study on Hate Speech Detection</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sneha</namePart>
<namePart type="family">Prashanth</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aradhana</namePart>
<namePart type="family">Mahati</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maksym</namePart>
<namePart type="family">Ziemlewski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicolas</namePart>
<namePart type="family">Loaiza</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Urja</namePart>
<namePart type="family">Khurana</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 10th Workshop on Online Abuse and Harms (WOAH)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Agostina</namePart>
<namePart type="family">Calabrese</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thomas</namePart>
<namePart type="family">Davidson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christine</namePart>
<namePart type="family">de Kock</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Urja</namePart>
<namePart type="family">Khurana</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marta</namePart>
<namePart type="family">Marchiori Manerba</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Paloma</namePart>
<namePart type="family">Piot</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zeerak</namePart>
<namePart type="family">Talat</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-487-3</identifier>
</relatedItem>
<abstract>Hate speech detection is inherently subjective because annotators routinely disagree on the same item, and collapsing their labels into a majority vote discards information about how contested the item actually is. Large language models exhibit variation by returning different labels for the same input across random seeds. We ask whether this variation empirically lines up with human disagreement. We operationalize annotator disagreement via entropy and the LLM instability via flip rate and model entropy across five seeds at fixed temperature, then measure their rank correlation on the full CREHate dataset using three models from both open- and closed-source families. The alignment with human disagreement is model dependent: Gemini 2.5 Flash and Llama 3.1 8B show significant positive correlations, while Mistral 7B shows only a marginal effect. The relationship is also modest in absolute scale: most items cluster at a flip rate of zero, and the signal is concentrated in a small fraction of high-disagreement items that also produce model instability. Qualitative analysis shows that culturally contested items, where human review is needed, sometimes produce zero model instability, as models classify them in opposite directions across cultural contexts. Our findings do not show evidence that seed-based instability is a reliable proxy for human disagreement for all LLMs.</abstract>
<identifier type="citekey">prashanth-etal-2026-llm</identifier>
<location>
<url>https://aclanthology.org/2026.woah-1.38/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>466</start>
<end>476</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Does LLM Instability Reflect Annotator Disagreement? A Study on Hate Speech Detection
%A Prashanth, Sneha
%A Mahati, Aradhana
%A Ziemlewski, Maksym
%A Loaiza, Nicolas
%A Khurana, Urja
%Y Calabrese, Agostina
%Y Davidson, Thomas
%Y de Kock, Christine
%Y Khurana, Urja
%Y Marchiori Manerba, Marta
%Y Piot, Paloma
%Y Talat, Zeerak
%S Proceedings of the 10th Workshop on Online Abuse and Harms (WOAH)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-487-3
%F prashanth-etal-2026-llm
%X Hate speech detection is inherently subjective because annotators routinely disagree on the same item, and collapsing their labels into a majority vote discards information about how contested the item actually is. Large language models exhibit variation by returning different labels for the same input across random seeds. We ask whether this variation empirically lines up with human disagreement. We operationalize annotator disagreement via entropy and the LLM instability via flip rate and model entropy across five seeds at fixed temperature, then measure their rank correlation on the full CREHate dataset using three models from both open- and closed-source families. The alignment with human disagreement is model dependent: Gemini 2.5 Flash and Llama 3.1 8B show significant positive correlations, while Mistral 7B shows only a marginal effect. The relationship is also modest in absolute scale: most items cluster at a flip rate of zero, and the signal is concentrated in a small fraction of high-disagreement items that also produce model instability. Qualitative analysis shows that culturally contested items, where human review is needed, sometimes produce zero model instability, as models classify them in opposite directions across cultural contexts. Our findings do not show evidence that seed-based instability is a reliable proxy for human disagreement for all LLMs.
%U https://aclanthology.org/2026.woah-1.38/
%P 466-476
Markdown (Informal)
[Does LLM Instability Reflect Annotator Disagreement? A Study on Hate Speech Detection](https://aclanthology.org/2026.woah-1.38/) (Prashanth et al., WOAH 2026)
ACL