@inproceedings{hossain-etal-2026-systematic,
title = "Systematic Failure Analysis of Open-Weight {LLM} Annotators for Classification Tasks: Taxonomy, Uncertainty Signals, and Deployment Boundaries",
author = "Hossain, Nahid and
Karip, Siam Rahman and
Kabir, Md Faisal and
Shatabda, Swakkhar and
Islam, Salekul",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2037/",
pages = "46409--46466",
ISBN = "979-8-89176-459-0",
abstract = "Large-scale annotated datasets underpin progress in natural language understanding, yet human annotation remains a persistent bottleneck. Large Language Models promise to bridge this gap, but their reliability as annotators is poorly understood. We evaluate 14 open-weight instruction-tuned models and 5 matched base{--}instruct pairs across 51,189 instances spanning 10 heterogeneous datasets, revealing that annotation failures are structurally predictable rather than random. We identify three recurring failure modes: Safety Hallucination, Intensity Collapse, and Entailment Bias. Safety hallucination can precede alignment rather than being introduced by it: some base models flag up to 100{\%} of benign text as hate speech. Alignment benefits scale inversely with base model capability, yielding a +21.6pp gain for the weakest base model yet a negligible +0.5pp shift for the strongest. KL divergence between base and instruct distributions is 2.0x higher on incorrect predictions, providing a lightweight instance-level signal for alignment-associated errors. Inter-prompt unanimity achieves 72{--}94{\%} accuracy at 71{--}90{\%} coverage on clean tasks, yet can actively degrade performance on tasks dominated by shared systematic bias. We translate these findings into a practical deployment framework with pre-screening diagnostics and tiered quality-routing policies, establishing principled boundaries for when LLM annotation can be trusted and when human intervention remains necessary."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hossain-etal-2026-systematic">
<titleInfo>
<title>Systematic Failure Analysis of Open-Weight LLM Annotators for Classification Tasks: Taxonomy, Uncertainty Signals, and Deployment Boundaries</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nahid</namePart>
<namePart type="family">Hossain</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siam</namePart>
<namePart type="given">Rahman</namePart>
<namePart type="family">Karip</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Md</namePart>
<namePart type="given">Faisal</namePart>
<namePart type="family">Kabir</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Swakkhar</namePart>
<namePart type="family">Shatabda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Salekul</namePart>
<namePart type="family">Islam</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large-scale annotated datasets underpin progress in natural language understanding, yet human annotation remains a persistent bottleneck. Large Language Models promise to bridge this gap, but their reliability as annotators is poorly understood. We evaluate 14 open-weight instruction-tuned models and 5 matched base–instruct pairs across 51,189 instances spanning 10 heterogeneous datasets, revealing that annotation failures are structurally predictable rather than random. We identify three recurring failure modes: Safety Hallucination, Intensity Collapse, and Entailment Bias. Safety hallucination can precede alignment rather than being introduced by it: some base models flag up to 100% of benign text as hate speech. Alignment benefits scale inversely with base model capability, yielding a +21.6pp gain for the weakest base model yet a negligible +0.5pp shift for the strongest. KL divergence between base and instruct distributions is 2.0x higher on incorrect predictions, providing a lightweight instance-level signal for alignment-associated errors. Inter-prompt unanimity achieves 72–94% accuracy at 71–90% coverage on clean tasks, yet can actively degrade performance on tasks dominated by shared systematic bias. We translate these findings into a practical deployment framework with pre-screening diagnostics and tiered quality-routing policies, establishing principled boundaries for when LLM annotation can be trusted and when human intervention remains necessary.</abstract>
<identifier type="citekey">hossain-etal-2026-systematic</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2037/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>46409</start>
<end>46466</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Systematic Failure Analysis of Open-Weight LLM Annotators for Classification Tasks: Taxonomy, Uncertainty Signals, and Deployment Boundaries
%A Hossain, Nahid
%A Karip, Siam Rahman
%A Kabir, Md Faisal
%A Shatabda, Swakkhar
%A Islam, Salekul
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F hossain-etal-2026-systematic
%X Large-scale annotated datasets underpin progress in natural language understanding, yet human annotation remains a persistent bottleneck. Large Language Models promise to bridge this gap, but their reliability as annotators is poorly understood. We evaluate 14 open-weight instruction-tuned models and 5 matched base–instruct pairs across 51,189 instances spanning 10 heterogeneous datasets, revealing that annotation failures are structurally predictable rather than random. We identify three recurring failure modes: Safety Hallucination, Intensity Collapse, and Entailment Bias. Safety hallucination can precede alignment rather than being introduced by it: some base models flag up to 100% of benign text as hate speech. Alignment benefits scale inversely with base model capability, yielding a +21.6pp gain for the weakest base model yet a negligible +0.5pp shift for the strongest. KL divergence between base and instruct distributions is 2.0x higher on incorrect predictions, providing a lightweight instance-level signal for alignment-associated errors. Inter-prompt unanimity achieves 72–94% accuracy at 71–90% coverage on clean tasks, yet can actively degrade performance on tasks dominated by shared systematic bias. We translate these findings into a practical deployment framework with pre-screening diagnostics and tiered quality-routing policies, establishing principled boundaries for when LLM annotation can be trusted and when human intervention remains necessary.
%U https://aclanthology.org/2026.emnlp-main.2037/
%P 46409-46466
Markdown (Informal)
[Systematic Failure Analysis of Open-Weight LLM Annotators for Classification Tasks: Taxonomy, Uncertainty Signals, and Deployment Boundaries](https://aclanthology.org/2026.emnlp-main.2037/) (Hossain et al., EMNLP 2026)
ACL