@inproceedings{roccabruna-etal-2026-mateo,
title = "{MATEO}: A Multimodal Benchmark for Planning with Temporal Execution Ordering",
author = "Roccabruna, Gabriel and
Khomyn, Olha and
Riccardi, Giuseppe",
editor = "Kamalloo, Ehsan and
Gontier, Nicolas and
Rozanov, Nikolai and
Hashemzadeh, Maryam and
Arora, Akhil",
booktitle = "Proceedings of the Second Workshop for {RE}search on Agent Language Models ({REALM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.realm-1.103/",
pages = "1501--1533",
ISBN = "979-8-89176-485-9",
abstract = "To achieve complex real-world goals, AI agents must be able to plan appropriate sequences of actions from a task specification. However, evaluating and improving the planning capabilities is challenging because planning involves multiple steps that are difficult to disentangle when viewed as a whole. A more feasible approach is to focus on Temporal Execution Order (TEO), a core planning sub-task that requires partially ordering actions based on dependency by reasoning about temporal and causal dependency constraints. Existing research studying models' reasoning over temporal execution is limited to automatically derived annotations, approximations of TEO as a linear chain, or text-only inputs. To address this gap, we introduce MATEO (MultimodAl Temporal Execution Order), a multimodal benchmark designed to assess and improve the temporal reasoning abilities of Large Vision Language Models (LVLMs). We acquire a high-quality multimodal recipe corpus, authored through a standardized editorial process that decomposes instructions into discrete steps, each paired with corresponding images. We design a novel set of guidelines and procedures to collect TEO annotations as graphs via crowdsourcing, which can be extended to new recipes, languages, and domains. Using MATEO, we evaluate six state-of-the-art LVLMs across model scales, varying language context, multimodal input structure, and fine-tuning strategies, showing the importance of multimodal inputs as well as limitations of these models."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="roccabruna-etal-2026-mateo">
<titleInfo>
<title>MATEO: A Multimodal Benchmark for Planning with Temporal Execution Ordering</title>
</titleInfo>
<name type="personal">
<namePart type="given">Gabriel</namePart>
<namePart type="family">Roccabruna</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Olha</namePart>
<namePart type="family">Khomyn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Giuseppe</namePart>
<namePart type="family">Riccardi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ehsan</namePart>
<namePart type="family">Kamalloo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicolas</namePart>
<namePart type="family">Gontier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikolai</namePart>
<namePart type="family">Rozanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maryam</namePart>
<namePart type="family">Hashemzadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akhil</namePart>
<namePart type="family">Arora</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-485-9</identifier>
</relatedItem>
<abstract>To achieve complex real-world goals, AI agents must be able to plan appropriate sequences of actions from a task specification. However, evaluating and improving the planning capabilities is challenging because planning involves multiple steps that are difficult to disentangle when viewed as a whole. A more feasible approach is to focus on Temporal Execution Order (TEO), a core planning sub-task that requires partially ordering actions based on dependency by reasoning about temporal and causal dependency constraints. Existing research studying models’ reasoning over temporal execution is limited to automatically derived annotations, approximations of TEO as a linear chain, or text-only inputs. To address this gap, we introduce MATEO (MultimodAl Temporal Execution Order), a multimodal benchmark designed to assess and improve the temporal reasoning abilities of Large Vision Language Models (LVLMs). We acquire a high-quality multimodal recipe corpus, authored through a standardized editorial process that decomposes instructions into discrete steps, each paired with corresponding images. We design a novel set of guidelines and procedures to collect TEO annotations as graphs via crowdsourcing, which can be extended to new recipes, languages, and domains. Using MATEO, we evaluate six state-of-the-art LVLMs across model scales, varying language context, multimodal input structure, and fine-tuning strategies, showing the importance of multimodal inputs as well as limitations of these models.</abstract>
<identifier type="citekey">roccabruna-etal-2026-mateo</identifier>
<location>
<url>https://aclanthology.org/2026.realm-1.103/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1501</start>
<end>1533</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MATEO: A Multimodal Benchmark for Planning with Temporal Execution Ordering
%A Roccabruna, Gabriel
%A Khomyn, Olha
%A Riccardi, Giuseppe
%Y Kamalloo, Ehsan
%Y Gontier, Nicolas
%Y Rozanov, Nikolai
%Y Hashemzadeh, Maryam
%Y Arora, Akhil
%S Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-485-9
%F roccabruna-etal-2026-mateo
%X To achieve complex real-world goals, AI agents must be able to plan appropriate sequences of actions from a task specification. However, evaluating and improving the planning capabilities is challenging because planning involves multiple steps that are difficult to disentangle when viewed as a whole. A more feasible approach is to focus on Temporal Execution Order (TEO), a core planning sub-task that requires partially ordering actions based on dependency by reasoning about temporal and causal dependency constraints. Existing research studying models’ reasoning over temporal execution is limited to automatically derived annotations, approximations of TEO as a linear chain, or text-only inputs. To address this gap, we introduce MATEO (MultimodAl Temporal Execution Order), a multimodal benchmark designed to assess and improve the temporal reasoning abilities of Large Vision Language Models (LVLMs). We acquire a high-quality multimodal recipe corpus, authored through a standardized editorial process that decomposes instructions into discrete steps, each paired with corresponding images. We design a novel set of guidelines and procedures to collect TEO annotations as graphs via crowdsourcing, which can be extended to new recipes, languages, and domains. Using MATEO, we evaluate six state-of-the-art LVLMs across model scales, varying language context, multimodal input structure, and fine-tuning strategies, showing the importance of multimodal inputs as well as limitations of these models.
%U https://aclanthology.org/2026.realm-1.103/
%P 1501-1533
Markdown (Informal)
[MATEO: A Multimodal Benchmark for Planning with Temporal Execution Ordering](https://aclanthology.org/2026.realm-1.103/) (Roccabruna et al., REALM 2026)
ACL