@inproceedings{torgbi-etal-2026-lingo,
title = "Lingo-Blocksworld: A Dataset for Linguistically-Informed Blocksworld",
author = "Torgbi, Melissa and
Post, Claire Benet and
Bonial, Claire and
Tayyar Madabushi, Harish",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.552/",
pages = "11626--11654",
ISBN = "979-8-89176-462-0",
abstract = "We introduce Lingo-Blocksworld, a linguistically-informed diagnostic benchmark for evaluating the generalisation and planning abilities of Large Language Models (LLMs) in the Blocksworld (BW) domain. Unlike prior obfuscated BW variants, our method replaces content words with morphosyntactically well-formed nonsense words, preserving the grammatical scaffolding of the domain while removing lexical familiarity. This enables us to test a psycholinguistically-motivated hypothesis, and find systematic support across both reasoning and non-reasoning models, with degradation patterns that mirror syntactic bootstrapping from human language processing. Additionally, we test the ability of models to generalise planning solutions from the typical 4-block problems to 12 blocks. Overall, our results demonstrate that reasoning models stay robust where non-reasoning models collapse: non-reasoning model performance drops to near zero once problems exceed four blocks or have actions and predicates obfuscated, while reasoning models maintain strong performance until structural and lexical difficulty are combined. Manual error analysis pinpoints further differences. Reasoning model errors are more often skipped steps or incomplete instruction following, whereas non-reasoning models fail to derive plausible plans at all. By combining psycholinguistic motivation with classical planning evaluation, Lingo-BW allows us to shift research focus from whether LLMs plan to how their planning depends on linguistic surface form."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="torgbi-etal-2026-lingo">
<titleInfo>
<title>Lingo-Blocksworld: A Dataset for Linguistically-Informed Blocksworld</title>
</titleInfo>
<name type="personal">
<namePart type="given">Melissa</namePart>
<namePart type="family">Torgbi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Claire</namePart>
<namePart type="given">Benet</namePart>
<namePart type="family">Post</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Claire</namePart>
<namePart type="family">Bonial</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Harish</namePart>
<namePart type="family">Tayyar Madabushi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>We introduce Lingo-Blocksworld, a linguistically-informed diagnostic benchmark for evaluating the generalisation and planning abilities of Large Language Models (LLMs) in the Blocksworld (BW) domain. Unlike prior obfuscated BW variants, our method replaces content words with morphosyntactically well-formed nonsense words, preserving the grammatical scaffolding of the domain while removing lexical familiarity. This enables us to test a psycholinguistically-motivated hypothesis, and find systematic support across both reasoning and non-reasoning models, with degradation patterns that mirror syntactic bootstrapping from human language processing. Additionally, we test the ability of models to generalise planning solutions from the typical 4-block problems to 12 blocks. Overall, our results demonstrate that reasoning models stay robust where non-reasoning models collapse: non-reasoning model performance drops to near zero once problems exceed four blocks or have actions and predicates obfuscated, while reasoning models maintain strong performance until structural and lexical difficulty are combined. Manual error analysis pinpoints further differences. Reasoning model errors are more often skipped steps or incomplete instruction following, whereas non-reasoning models fail to derive plausible plans at all. By combining psycholinguistic motivation with classical planning evaluation, Lingo-BW allows us to shift research focus from whether LLMs plan to how their planning depends on linguistic surface form.</abstract>
<identifier type="citekey">torgbi-etal-2026-lingo</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.552/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>11626</start>
<end>11654</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Lingo-Blocksworld: A Dataset for Linguistically-Informed Blocksworld
%A Torgbi, Melissa
%A Post, Claire Benet
%A Bonial, Claire
%A Tayyar Madabushi, Harish
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F torgbi-etal-2026-lingo
%X We introduce Lingo-Blocksworld, a linguistically-informed diagnostic benchmark for evaluating the generalisation and planning abilities of Large Language Models (LLMs) in the Blocksworld (BW) domain. Unlike prior obfuscated BW variants, our method replaces content words with morphosyntactically well-formed nonsense words, preserving the grammatical scaffolding of the domain while removing lexical familiarity. This enables us to test a psycholinguistically-motivated hypothesis, and find systematic support across both reasoning and non-reasoning models, with degradation patterns that mirror syntactic bootstrapping from human language processing. Additionally, we test the ability of models to generalise planning solutions from the typical 4-block problems to 12 blocks. Overall, our results demonstrate that reasoning models stay robust where non-reasoning models collapse: non-reasoning model performance drops to near zero once problems exceed four blocks or have actions and predicates obfuscated, while reasoning models maintain strong performance until structural and lexical difficulty are combined. Manual error analysis pinpoints further differences. Reasoning model errors are more often skipped steps or incomplete instruction following, whereas non-reasoning models fail to derive plausible plans at all. By combining psycholinguistic motivation with classical planning evaluation, Lingo-BW allows us to shift research focus from whether LLMs plan to how their planning depends on linguistic surface form.
%U https://aclanthology.org/2026.findings-emnlp.552/
%P 11626-11654
Markdown (Informal)
[Lingo-Blocksworld: A Dataset for Linguistically-Informed Blocksworld](https://aclanthology.org/2026.findings-emnlp.552/) (Torgbi et al., Findings 2026)
ACL