@inproceedings{schorr-etal-2026-humans,
title = "When Humans and {LLM}s Disagree: Understanding Human-{LLM} Disagreement in Learner Intent Annotation for Educational Chatbot Dialogues",
author = "Schorr, Isabel and
Bardach, Lisa and
Gao, Haoxiang and
Kasneci, Enkelejda",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.499/",
pages = "11462--11487",
ISBN = "979-8-89176-459-0",
abstract = "As large language model (LLM) chatbots become embedded in education, researchers need scalable ways to analyze what learners are actually doing in student-chatbot interactions. To address this challenge, we propose a learning theory-informed annotation framework for learner intent and engagement in educational chatbot dialogues, operationalizing solution seeking, knowledge seeking, cognitive engagement, and mental model expression through a hierarchical decision process, and apply it manually to a dataset of 2,393 learner-chatbot interactions (408 double-coded). We then evaluate two LLMs as zero-shot annotators and compare human-human, LLM-LLM, and human-LLM agreement across both intermediate decisions and final intent labels. While the two LLMs exhibit agreement patterns approaching human-human reliability on the final label (human {\ensuremath{\kappa}} = 0.73 vs. LLM {\ensuremath{\kappa}} = 0.67), their agreement with human reference labels is substantially lower ({\ensuremath{\kappa}} {\ensuremath{\leq}} 0.53), indicating that automated annotation errors cannot be understood from aggregate scores alone. Through exploratory qualitative coding of disagreement cases, we find that divergences can arise from ambiguous learner intent, competing valid interpretations, and human annotation variability, rather than model failure alone."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="schorr-etal-2026-humans">
<titleInfo>
<title>When Humans and LLMs Disagree: Understanding Human-LLM Disagreement in Learner Intent Annotation for Educational Chatbot Dialogues</title>
</titleInfo>
<name type="personal">
<namePart type="given">Isabel</namePart>
<namePart type="family">Schorr</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lisa</namePart>
<namePart type="family">Bardach</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haoxiang</namePart>
<namePart type="family">Gao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Enkelejda</namePart>
<namePart type="family">Kasneci</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>As large language model (LLM) chatbots become embedded in education, researchers need scalable ways to analyze what learners are actually doing in student-chatbot interactions. To address this challenge, we propose a learning theory-informed annotation framework for learner intent and engagement in educational chatbot dialogues, operationalizing solution seeking, knowledge seeking, cognitive engagement, and mental model expression through a hierarchical decision process, and apply it manually to a dataset of 2,393 learner-chatbot interactions (408 double-coded). We then evaluate two LLMs as zero-shot annotators and compare human-human, LLM-LLM, and human-LLM agreement across both intermediate decisions and final intent labels. While the two LLMs exhibit agreement patterns approaching human-human reliability on the final label (human \ensuremathąppa = 0.73 vs. LLM \ensuremathąppa = 0.67), their agreement with human reference labels is substantially lower (\ensuremathąppa \ensuremathłeq 0.53), indicating that automated annotation errors cannot be understood from aggregate scores alone. Through exploratory qualitative coding of disagreement cases, we find that divergences can arise from ambiguous learner intent, competing valid interpretations, and human annotation variability, rather than model failure alone.</abstract>
<identifier type="citekey">schorr-etal-2026-humans</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.499/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>11462</start>
<end>11487</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Humans and LLMs Disagree: Understanding Human-LLM Disagreement in Learner Intent Annotation for Educational Chatbot Dialogues
%A Schorr, Isabel
%A Bardach, Lisa
%A Gao, Haoxiang
%A Kasneci, Enkelejda
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F schorr-etal-2026-humans
%X As large language model (LLM) chatbots become embedded in education, researchers need scalable ways to analyze what learners are actually doing in student-chatbot interactions. To address this challenge, we propose a learning theory-informed annotation framework for learner intent and engagement in educational chatbot dialogues, operationalizing solution seeking, knowledge seeking, cognitive engagement, and mental model expression through a hierarchical decision process, and apply it manually to a dataset of 2,393 learner-chatbot interactions (408 double-coded). We then evaluate two LLMs as zero-shot annotators and compare human-human, LLM-LLM, and human-LLM agreement across both intermediate decisions and final intent labels. While the two LLMs exhibit agreement patterns approaching human-human reliability on the final label (human \ensuremathąppa = 0.73 vs. LLM \ensuremathąppa = 0.67), their agreement with human reference labels is substantially lower (\ensuremathąppa \ensuremathłeq 0.53), indicating that automated annotation errors cannot be understood from aggregate scores alone. Through exploratory qualitative coding of disagreement cases, we find that divergences can arise from ambiguous learner intent, competing valid interpretations, and human annotation variability, rather than model failure alone.
%U https://aclanthology.org/2026.emnlp-main.499/
%P 11462-11487
Markdown (Informal)
[When Humans and LLMs Disagree: Understanding Human-LLM Disagreement in Learner Intent Annotation for Educational Chatbot Dialogues](https://aclanthology.org/2026.emnlp-main.499/) (Schorr et al., EMNLP 2026)
ACL