@inproceedings{kosterna-etal-2026-disentangling,
title = "Disentangling Form and World Knowledge in {LLM} Interpretation: Evidence from Quantifier Scope Disambiguation",
author = "Kosterna, Jakub and
Grudzi{\'n}ska, Justyna and
Miecznikowski, Maciej and
Borysewicz, Wojciech and
Poteralska, Julia and
Rutkowski, Kacper and
Kwapisz, Jan Henryk",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.985/",
doi = "10.18653/2026.findings-emnlp.985",
pages = "20823--20833",
ISBN = "979-8-89176-462-0",
abstract = "We investigate how large language models (LLMs) construct meaning using Quantifier Scope Disambiguation (QSD) as a controlled probing task. We introduce (i) a balanced English corpus designed to neutralize classical surface heuristics and systematically manipulate the relationship between formal predictors and preferred readings, and (ii) a pseudosentence dataset that attenuates world-knowledge cues while preserving formal information relevant to scope. In Experiment 1, we evaluate a range of LLMs in a zero-shot question-answering setup and compare them to human baselines. While models achieve high accuracy on the balanced corpus, performance drops substantially on pseudosentences. Crucially, robust performance is retained only when classical formal predictors converge on the preferred reading; once these predictors partially diverge or fully conflict, performance approaches chance levels and becomes highly unstable across models. These findings suggest that the strong QSD performance reported for current LLMs may reflect a masking effect caused by the systematic alignment of surface-level heuristics and world knowledge in naturalistic datasets. As an exploratory extension, Experiment 2 examines whether adding externally retrieved context changes model behavior while keeping the task and prompt fixed. RAG has a negligible aggregate effect, with the largest positive change occurring in the hardest configuration, where surface cues are maximally misleading. Taken together, our results suggest that high QSD performance in LLMs remains strongly dependent on convergence between formal cues and world knowledge, despite the apparent semantic robustness suggested by naturalistic QSD results."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kosterna-etal-2026-disentangling">
<titleInfo>
<title>Disentangling Form and World Knowledge in LLM Interpretation: Evidence from Quantifier Scope Disambiguation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jakub</namePart>
<namePart type="family">Kosterna</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Justyna</namePart>
<namePart type="family">Grudzińska</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maciej</namePart>
<namePart type="family">Miecznikowski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wojciech</namePart>
<namePart type="family">Borysewicz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Julia</namePart>
<namePart type="family">Poteralska</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kacper</namePart>
<namePart type="family">Rutkowski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jan</namePart>
<namePart type="given">Henryk</namePart>
<namePart type="family">Kwapisz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>We investigate how large language models (LLMs) construct meaning using Quantifier Scope Disambiguation (QSD) as a controlled probing task. We introduce (i) a balanced English corpus designed to neutralize classical surface heuristics and systematically manipulate the relationship between formal predictors and preferred readings, and (ii) a pseudosentence dataset that attenuates world-knowledge cues while preserving formal information relevant to scope. In Experiment 1, we evaluate a range of LLMs in a zero-shot question-answering setup and compare them to human baselines. While models achieve high accuracy on the balanced corpus, performance drops substantially on pseudosentences. Crucially, robust performance is retained only when classical formal predictors converge on the preferred reading; once these predictors partially diverge or fully conflict, performance approaches chance levels and becomes highly unstable across models. These findings suggest that the strong QSD performance reported for current LLMs may reflect a masking effect caused by the systematic alignment of surface-level heuristics and world knowledge in naturalistic datasets. As an exploratory extension, Experiment 2 examines whether adding externally retrieved context changes model behavior while keeping the task and prompt fixed. RAG has a negligible aggregate effect, with the largest positive change occurring in the hardest configuration, where surface cues are maximally misleading. Taken together, our results suggest that high QSD performance in LLMs remains strongly dependent on convergence between formal cues and world knowledge, despite the apparent semantic robustness suggested by naturalistic QSD results.</abstract>
<identifier type="citekey">kosterna-etal-2026-disentangling</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.985</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.985/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>20823</start>
<end>20833</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Disentangling Form and World Knowledge in LLM Interpretation: Evidence from Quantifier Scope Disambiguation
%A Kosterna, Jakub
%A Grudzińska, Justyna
%A Miecznikowski, Maciej
%A Borysewicz, Wojciech
%A Poteralska, Julia
%A Rutkowski, Kacper
%A Kwapisz, Jan Henryk
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F kosterna-etal-2026-disentangling
%X We investigate how large language models (LLMs) construct meaning using Quantifier Scope Disambiguation (QSD) as a controlled probing task. We introduce (i) a balanced English corpus designed to neutralize classical surface heuristics and systematically manipulate the relationship between formal predictors and preferred readings, and (ii) a pseudosentence dataset that attenuates world-knowledge cues while preserving formal information relevant to scope. In Experiment 1, we evaluate a range of LLMs in a zero-shot question-answering setup and compare them to human baselines. While models achieve high accuracy on the balanced corpus, performance drops substantially on pseudosentences. Crucially, robust performance is retained only when classical formal predictors converge on the preferred reading; once these predictors partially diverge or fully conflict, performance approaches chance levels and becomes highly unstable across models. These findings suggest that the strong QSD performance reported for current LLMs may reflect a masking effect caused by the systematic alignment of surface-level heuristics and world knowledge in naturalistic datasets. As an exploratory extension, Experiment 2 examines whether adding externally retrieved context changes model behavior while keeping the task and prompt fixed. RAG has a negligible aggregate effect, with the largest positive change occurring in the hardest configuration, where surface cues are maximally misleading. Taken together, our results suggest that high QSD performance in LLMs remains strongly dependent on convergence between formal cues and world knowledge, despite the apparent semantic robustness suggested by naturalistic QSD results.
%R 10.18653/2026.findings-emnlp.985
%U https://aclanthology.org/2026.findings-emnlp.985/
%U https://doi.org/10.18653/2026.findings-emnlp.985
%P 20823-20833
Markdown (Informal)
[Disentangling Form and World Knowledge in LLM Interpretation: Evidence from Quantifier Scope Disambiguation](https://aclanthology.org/2026.findings-emnlp.985/) (Kosterna et al., Findings 2026)
ACL