@inproceedings{yerukola-etal-2026-xybench,
title = "{XYB}ench: Can {LLM}s Respond Pragmatically to Queries with Misconceptions?",
author = "Yerukola, Akhila and
Hwang, Jena D. and
Zheng, Mingqian and
Godsey, Jenna and
Kim, Hyunwoo and
Pyatkin, Valentina and
Hu, Jennifer and
Sap, Maarten",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2646/",
pages = "60203--60234",
ISBN = "979-8-89176-459-0",
abstract = "When non-expert users ask LLMs for assistance, their queries can often have misconceptions (e.g., ``How do I parse XML with regex?''). In such cases, often referred to as the XY-problem, LLMs must identify the misconception ({''}regex are fragile'') and meaningfully direct the user toward a pragmatic solution that will address the root problem implicit in the request ({''}use an XML parser''). We introduce XYBench, a benchmark of 8,115 such queries, drawn from technical (StackOverflow/StackExchange) and everyday (WikiHow and a manually-curated subset) domains. We design an evaluation paradigm that assesses model responses along three criteria grounded in cooperative response theory: (a) presence and (b) emphasis on pragmatic solutions, and (c) identification of misconceptions. Our experiments show that even the strongest LLMs predominantly answer the literal request (0.75{--}0.92) and far less often the intended one (0.33{--}0.71), while substantially lagging behind humans at identifying misconceptions (at most 63{\%} vs. 79{--}90{\%}). Further, models overwhelmingly prefer pragmatic responses in a multiple choice setting yet consistently fail to generate them. Oracle ablation experiments show that providing explicit user intent at generation time helps; however a large gap remains, suggesting pragmatic redirection is a fundamentally underdeveloped capability in current LLMs."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yerukola-etal-2026-xybench">
<titleInfo>
<title>XYBench: Can LLMs Respond Pragmatically to Queries with Misconceptions?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Akhila</namePart>
<namePart type="family">Yerukola</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jena</namePart>
<namePart type="given">D</namePart>
<namePart type="family">Hwang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mingqian</namePart>
<namePart type="family">Zheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jenna</namePart>
<namePart type="family">Godsey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hyunwoo</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Valentina</namePart>
<namePart type="family">Pyatkin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jennifer</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maarten</namePart>
<namePart type="family">Sap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>When non-expert users ask LLMs for assistance, their queries can often have misconceptions (e.g., “How do I parse XML with regex?”). In such cases, often referred to as the XY-problem, LLMs must identify the misconception (”regex are fragile”) and meaningfully direct the user toward a pragmatic solution that will address the root problem implicit in the request (”use an XML parser”). We introduce XYBench, a benchmark of 8,115 such queries, drawn from technical (StackOverflow/StackExchange) and everyday (WikiHow and a manually-curated subset) domains. We design an evaluation paradigm that assesses model responses along three criteria grounded in cooperative response theory: (a) presence and (b) emphasis on pragmatic solutions, and (c) identification of misconceptions. Our experiments show that even the strongest LLMs predominantly answer the literal request (0.75–0.92) and far less often the intended one (0.33–0.71), while substantially lagging behind humans at identifying misconceptions (at most 63% vs. 79–90%). Further, models overwhelmingly prefer pragmatic responses in a multiple choice setting yet consistently fail to generate them. Oracle ablation experiments show that providing explicit user intent at generation time helps; however a large gap remains, suggesting pragmatic redirection is a fundamentally underdeveloped capability in current LLMs.</abstract>
<identifier type="citekey">yerukola-etal-2026-xybench</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2646/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>60203</start>
<end>60234</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T XYBench: Can LLMs Respond Pragmatically to Queries with Misconceptions?
%A Yerukola, Akhila
%A Hwang, Jena D.
%A Zheng, Mingqian
%A Godsey, Jenna
%A Kim, Hyunwoo
%A Pyatkin, Valentina
%A Hu, Jennifer
%A Sap, Maarten
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F yerukola-etal-2026-xybench
%X When non-expert users ask LLMs for assistance, their queries can often have misconceptions (e.g., “How do I parse XML with regex?”). In such cases, often referred to as the XY-problem, LLMs must identify the misconception (”regex are fragile”) and meaningfully direct the user toward a pragmatic solution that will address the root problem implicit in the request (”use an XML parser”). We introduce XYBench, a benchmark of 8,115 such queries, drawn from technical (StackOverflow/StackExchange) and everyday (WikiHow and a manually-curated subset) domains. We design an evaluation paradigm that assesses model responses along three criteria grounded in cooperative response theory: (a) presence and (b) emphasis on pragmatic solutions, and (c) identification of misconceptions. Our experiments show that even the strongest LLMs predominantly answer the literal request (0.75–0.92) and far less often the intended one (0.33–0.71), while substantially lagging behind humans at identifying misconceptions (at most 63% vs. 79–90%). Further, models overwhelmingly prefer pragmatic responses in a multiple choice setting yet consistently fail to generate them. Oracle ablation experiments show that providing explicit user intent at generation time helps; however a large gap remains, suggesting pragmatic redirection is a fundamentally underdeveloped capability in current LLMs.
%U https://aclanthology.org/2026.emnlp-main.2646/
%P 60203-60234
Markdown (Informal)
[XYBench: Can LLMs Respond Pragmatically to Queries with Misconceptions?](https://aclanthology.org/2026.emnlp-main.2646/) (Yerukola et al., EMNLP 2026)
ACL
- Akhila Yerukola, Jena D. Hwang, Mingqian Zheng, Jenna Godsey, Hyunwoo Kim, Valentina Pyatkin, Jennifer Hu, and Maarten Sap. 2026. XYBench: Can LLMs Respond Pragmatically to Queries with Misconceptions?. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 60203–60234, Budapest, Hungary. Association for Computational Linguistics.