@inproceedings{ganesh-etal-2026-pilot,
title = "A Pilot Study Investigating Stakeholder Subjectivity in Collaborative Dialog Analysis",
author = "Ganesh, Ananya and
Palmer, Martha and
von der Wense, Katharina",
editor = "Dudy, Shiran and
Abercrombie, Gavin and
Basile, Valerio and
Leonardelli, Elisa and
Frenda, Simona",
booktitle = "Proceedings of the the fifth edition of {NLP}erspectives",
month = may,
year = "2026",
address = "Palma, Mallorca (Spain)",
publisher = "ELRA Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.nlperspectives-1.13/",
doi = "10.63317/52krmhen6sw2",
pages = "136--144",
abstract = "Qualitative research in education relies on{``}ground truth'' codes or labels generated by having a trained or expert coder code observations in data such as student dialog. Although rigorous validity checks are a part of the coding process, there is limited research investigating how and to what extent, this notion of the ground truth is influenced by inherent task subjectivity. This paper presents a pilot study of task subjectivity centered around the phenomenon of verbal off-task behavior. The context for this study is real-world small-group collaborative conversations among three to five students in a middle-school science classroom. To investigate how stakeholders such as teachers and students show subjectivity in approaching this task, we recruit five teachers from the Prolific online platform, and five students from local middle and high schools as annotators of off-task speech. We show that teachers, students, and expert coders differ in their perception of off-task speech, with some of these differences being systematic. Drawing upon recent research in machine learning and natural language processing, we then outline the potential benefits of collecting and modeling a range of codes that explicitly represent the subjective perspectives of a diverse set of coders."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ganesh-etal-2026-pilot">
<titleInfo>
<title>A Pilot Study Investigating Stakeholder Subjectivity in Collaborative Dialog Analysis</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ananya</namePart>
<namePart type="family">Ganesh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Martha</namePart>
<namePart type="family">Palmer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Katharina</namePart>
<namePart type="family">von der Wense</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the the fifth edition of NLPerspectives</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shiran</namePart>
<namePart type="family">Dudy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gavin</namePart>
<namePart type="family">Abercrombie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Valerio</namePart>
<namePart type="family">Basile</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elisa</namePart>
<namePart type="family">Leonardelli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simona</namePart>
<namePart type="family">Frenda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma, Mallorca (Spain)</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Qualitative research in education relies on“ground truth” codes or labels generated by having a trained or expert coder code observations in data such as student dialog. Although rigorous validity checks are a part of the coding process, there is limited research investigating how and to what extent, this notion of the ground truth is influenced by inherent task subjectivity. This paper presents a pilot study of task subjectivity centered around the phenomenon of verbal off-task behavior. The context for this study is real-world small-group collaborative conversations among three to five students in a middle-school science classroom. To investigate how stakeholders such as teachers and students show subjectivity in approaching this task, we recruit five teachers from the Prolific online platform, and five students from local middle and high schools as annotators of off-task speech. We show that teachers, students, and expert coders differ in their perception of off-task speech, with some of these differences being systematic. Drawing upon recent research in machine learning and natural language processing, we then outline the potential benefits of collecting and modeling a range of codes that explicitly represent the subjective perspectives of a diverse set of coders.</abstract>
<identifier type="citekey">ganesh-etal-2026-pilot</identifier>
<identifier type="doi">10.63317/52krmhen6sw2</identifier>
<location>
<url>https://aclanthology.org/2026.nlperspectives-1.13/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>136</start>
<end>144</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T A Pilot Study Investigating Stakeholder Subjectivity in Collaborative Dialog Analysis
%A Ganesh, Ananya
%A Palmer, Martha
%A von der Wense, Katharina
%Y Dudy, Shiran
%Y Abercrombie, Gavin
%Y Basile, Valerio
%Y Leonardelli, Elisa
%Y Frenda, Simona
%S Proceedings of the the fifth edition of NLPerspectives
%D 2026
%8 May
%I ELRA Language Resources Association (ELRA)
%C Palma, Mallorca (Spain)
%F ganesh-etal-2026-pilot
%X Qualitative research in education relies on“ground truth” codes or labels generated by having a trained or expert coder code observations in data such as student dialog. Although rigorous validity checks are a part of the coding process, there is limited research investigating how and to what extent, this notion of the ground truth is influenced by inherent task subjectivity. This paper presents a pilot study of task subjectivity centered around the phenomenon of verbal off-task behavior. The context for this study is real-world small-group collaborative conversations among three to five students in a middle-school science classroom. To investigate how stakeholders such as teachers and students show subjectivity in approaching this task, we recruit five teachers from the Prolific online platform, and five students from local middle and high schools as annotators of off-task speech. We show that teachers, students, and expert coders differ in their perception of off-task speech, with some of these differences being systematic. Drawing upon recent research in machine learning and natural language processing, we then outline the potential benefits of collecting and modeling a range of codes that explicitly represent the subjective perspectives of a diverse set of coders.
%R 10.63317/52krmhen6sw2
%U https://aclanthology.org/2026.nlperspectives-1.13/
%U https://doi.org/10.63317/52krmhen6sw2
%P 136-144
Markdown (Informal)
[A Pilot Study Investigating Stakeholder Subjectivity in Collaborative Dialog Analysis](https://aclanthology.org/2026.nlperspectives-1.13/) (Ganesh et al., NLPerspectives 2026)
ACL