@inproceedings{liu-etal-2026-tom,
title = "{T}o{M}-Video: A Theory-of-Mind Video Benchmark with Spatio-Temporal Grounded Reasoning",
author = "Liu, Quanling and
Lin, Zizhuo and
Quan, Jinsheng and
Shi, Xin and
Zhang, Xueqiao and
Zhang, Chao and
Zhu, Yifan and
Li, Zhihui and
Luo, Yawei",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.302/",
pages = "6944--6972",
ISBN = "979-8-89176-459-0",
abstract = "Humans can infer others' beliefs, intentions, and emotions from observable behavior, an ability known as Theory of Mind (ToM). While current Multimodal Large Language Models (MLLMs) excel at visual understanding, can they identify mental-state cues in videos and reason about evolving mental states? To systematically evaluate spatio-temporal ToM in MLLMs, we introduce ToM-Video, a large-scale benchmark built from diverse real-world narrative videos. Through multi-stage filtering and annotation from massive data sources, ToM-Video provides a high-quality collection of ToM-centric scenes, comprising 1,487 videos and 41K visual question answering (VQA) pairs. We further propose INSPIRE, a post-training framework that, through a carefully designed three-stage curriculum, progressively teaches models to locate ToM-relevant cues and connect them to latent mental-state changes. Experiments show that current MLLMs remain weak at linking observable cues to evolving mental states, while INSPIRE shifts the model from surface description toward compact, evidence-backed inference, leading to higher accuracy and better generalization to external social-reasoning video datasets."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-tom">
<titleInfo>
<title>ToM-Video: A Theory-of-Mind Video Benchmark with Spatio-Temporal Grounded Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Quanling</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zizhuo</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jinsheng</namePart>
<namePart type="family">Quan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Shi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xueqiao</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chao</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yifan</namePart>
<namePart type="family">Zhu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhihui</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yawei</namePart>
<namePart type="family">Luo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Humans can infer others’ beliefs, intentions, and emotions from observable behavior, an ability known as Theory of Mind (ToM). While current Multimodal Large Language Models (MLLMs) excel at visual understanding, can they identify mental-state cues in videos and reason about evolving mental states? To systematically evaluate spatio-temporal ToM in MLLMs, we introduce ToM-Video, a large-scale benchmark built from diverse real-world narrative videos. Through multi-stage filtering and annotation from massive data sources, ToM-Video provides a high-quality collection of ToM-centric scenes, comprising 1,487 videos and 41K visual question answering (VQA) pairs. We further propose INSPIRE, a post-training framework that, through a carefully designed three-stage curriculum, progressively teaches models to locate ToM-relevant cues and connect them to latent mental-state changes. Experiments show that current MLLMs remain weak at linking observable cues to evolving mental states, while INSPIRE shifts the model from surface description toward compact, evidence-backed inference, leading to higher accuracy and better generalization to external social-reasoning video datasets.</abstract>
<identifier type="citekey">liu-etal-2026-tom</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.302/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>6944</start>
<end>6972</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T ToM-Video: A Theory-of-Mind Video Benchmark with Spatio-Temporal Grounded Reasoning
%A Liu, Quanling
%A Lin, Zizhuo
%A Quan, Jinsheng
%A Shi, Xin
%A Zhang, Xueqiao
%A Zhang, Chao
%A Zhu, Yifan
%A Li, Zhihui
%A Luo, Yawei
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F liu-etal-2026-tom
%X Humans can infer others’ beliefs, intentions, and emotions from observable behavior, an ability known as Theory of Mind (ToM). While current Multimodal Large Language Models (MLLMs) excel at visual understanding, can they identify mental-state cues in videos and reason about evolving mental states? To systematically evaluate spatio-temporal ToM in MLLMs, we introduce ToM-Video, a large-scale benchmark built from diverse real-world narrative videos. Through multi-stage filtering and annotation from massive data sources, ToM-Video provides a high-quality collection of ToM-centric scenes, comprising 1,487 videos and 41K visual question answering (VQA) pairs. We further propose INSPIRE, a post-training framework that, through a carefully designed three-stage curriculum, progressively teaches models to locate ToM-relevant cues and connect them to latent mental-state changes. Experiments show that current MLLMs remain weak at linking observable cues to evolving mental states, while INSPIRE shifts the model from surface description toward compact, evidence-backed inference, leading to higher accuracy and better generalization to external social-reasoning video datasets.
%U https://aclanthology.org/2026.emnlp-main.302/
%P 6944-6972
Markdown (Informal)
[ToM-Video: A Theory-of-Mind Video Benchmark with Spatio-Temporal Grounded Reasoning](https://aclanthology.org/2026.emnlp-main.302/) (Liu et al., EMNLP 2026)
ACL
- Quanling Liu, Zizhuo Lin, Jinsheng Quan, Xin Shi, Xueqiao Zhang, Chao Zhang, Yifan Zhu, Zhihui Li, and Yawei Luo. 2026. ToM-Video: A Theory-of-Mind Video Benchmark with Spatio-Temporal Grounded Reasoning. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 6944–6972, Budapest, Hungary. Association for Computational Linguistics.