@inproceedings{kowalyshyn-scheutz-2026-llms,
title = "{LLM}s and their Limited Theory of Mind: Evaluating Mental State Annotations in Situated Dialogue",
author = "Kowalyshyn, Katharine and
Scheutz, Matthias",
editor = "Choi, Jinho D. and
Chen, Yun-Nung and
Funakoshi, Kotaro and
Emami, Ali",
booktitle = "Proceedings of the 27th Annual Meeting of the Special Interest Group on Discourse and Dialogue",
month = aug,
year = "2026",
address = "Atlanta, Georgia, USA",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.sigdial-1.52/",
pages = "738--756",
abstract = "Effective human teams excel at maintaining a consistent shared mental model (SMM) that reflects the shared understanding of individual team members about the task and what remains to be done. We present a novel, two-step framework that leverages large language models (LLMs) both as (1) annotators of team dialogues to track the team{'}s SMM and (2) as automated discrepancy detectors among individuals' mental states as they are represented in individual mental models. We define an SMM coherence evaluation framework for this use case and apply it to six dialogues in a previously published team corpus, ultimately producing a dataset of human and LLM SMM annotations, a reproducible evaluation framework for SMM coherence, and an empirical assessment of LLM-based discrepancy detection. Our results reveal that while LLMs exhibit apparent coherence on straightforward natural-language annotation tasks, they systematically err in scenarios requiring spatial reasoning or disambiguation of transcription-level disfluencies."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kowalyshyn-scheutz-2026-llms">
<titleInfo>
<title>LLMs and their Limited Theory of Mind: Evaluating Mental State Annotations in Situated Dialogue</title>
</titleInfo>
<name type="personal">
<namePart type="given">Katharine</namePart>
<namePart type="family">Kowalyshyn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Matthias</namePart>
<namePart type="family">Scheutz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-08</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 27th Annual Meeting of the Special Interest Group on Discourse and Dialogue</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jinho</namePart>
<namePart type="given">D</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yun-Nung</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kotaro</namePart>
<namePart type="family">Funakoshi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ali</namePart>
<namePart type="family">Emami</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Atlanta, Georgia, USA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Effective human teams excel at maintaining a consistent shared mental model (SMM) that reflects the shared understanding of individual team members about the task and what remains to be done. We present a novel, two-step framework that leverages large language models (LLMs) both as (1) annotators of team dialogues to track the team’s SMM and (2) as automated discrepancy detectors among individuals’ mental states as they are represented in individual mental models. We define an SMM coherence evaluation framework for this use case and apply it to six dialogues in a previously published team corpus, ultimately producing a dataset of human and LLM SMM annotations, a reproducible evaluation framework for SMM coherence, and an empirical assessment of LLM-based discrepancy detection. Our results reveal that while LLMs exhibit apparent coherence on straightforward natural-language annotation tasks, they systematically err in scenarios requiring spatial reasoning or disambiguation of transcription-level disfluencies.</abstract>
<identifier type="citekey">kowalyshyn-scheutz-2026-llms</identifier>
<location>
<url>https://aclanthology.org/2026.sigdial-1.52/</url>
</location>
<part>
<date>2026-08</date>
<extent unit="page">
<start>738</start>
<end>756</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T LLMs and their Limited Theory of Mind: Evaluating Mental State Annotations in Situated Dialogue
%A Kowalyshyn, Katharine
%A Scheutz, Matthias
%Y Choi, Jinho D.
%Y Chen, Yun-Nung
%Y Funakoshi, Kotaro
%Y Emami, Ali
%S Proceedings of the 27th Annual Meeting of the Special Interest Group on Discourse and Dialogue
%D 2026
%8 August
%I Association for Computational Linguistics
%C Atlanta, Georgia, USA
%F kowalyshyn-scheutz-2026-llms
%X Effective human teams excel at maintaining a consistent shared mental model (SMM) that reflects the shared understanding of individual team members about the task and what remains to be done. We present a novel, two-step framework that leverages large language models (LLMs) both as (1) annotators of team dialogues to track the team’s SMM and (2) as automated discrepancy detectors among individuals’ mental states as they are represented in individual mental models. We define an SMM coherence evaluation framework for this use case and apply it to six dialogues in a previously published team corpus, ultimately producing a dataset of human and LLM SMM annotations, a reproducible evaluation framework for SMM coherence, and an empirical assessment of LLM-based discrepancy detection. Our results reveal that while LLMs exhibit apparent coherence on straightforward natural-language annotation tasks, they systematically err in scenarios requiring spatial reasoning or disambiguation of transcription-level disfluencies.
%U https://aclanthology.org/2026.sigdial-1.52/
%P 738-756
Markdown (Informal)
[LLMs and their Limited Theory of Mind: Evaluating Mental State Annotations in Situated Dialogue](https://aclanthology.org/2026.sigdial-1.52/) (Kowalyshyn & Scheutz, SIGDIAL 2026)
ACL