@inproceedings{dai-2026-dont,
title = "You Don{'}t Understand: Benchmarking Responses to Experiential Mismatch in {LLM}s",
author = "Dai, Yuyang",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 2: Short Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-short.31/",
pages = "463--473",
ISBN = "979-8-89176-440-8",
abstract = "Current LLM evaluation frameworks primarily assess understanding through task performance, focusing on factual accuracy, reasoning, and instruction following, while overlooking the experiential dimensions of human comprehension. Models that produce fluent, empathetic responses score highly even when they claim experiential access they cannot possess. We introduce ComprehensionGap, a benchmark of 500 scenarios spanning five empirically grounded categories of experiential incommensurability: Embodied Qualia, Episodic Incommensurability, Cultural Tacit Knowledge, Relational Situatedness, and Existential Finitude. Using a human-judged protocol measuring Acknowledgement of Limit, Absence of False Empathy, and Situational Relevance, we evaluate six frontier and open-weight models. Our experiments reveal a consistent Grounding Gap: all models maintain high fluency while systematically overclaiming experiential understanding, with larger models producing more convincing yet less honest mimicry. Models can calibrate for information gaps but not experiential ones. These findings argue for integrating experiential calibration into future alignment and evaluation frameworks."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="dai-2026-dont">
<titleInfo>
<title>You Don’t Understand: Benchmarking Responses to Experiential Mismatch in LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yuyang</namePart>
<namePart type="family">Dai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-440-8</identifier>
</relatedItem>
<abstract>Current LLM evaluation frameworks primarily assess understanding through task performance, focusing on factual accuracy, reasoning, and instruction following, while overlooking the experiential dimensions of human comprehension. Models that produce fluent, empathetic responses score highly even when they claim experiential access they cannot possess. We introduce ComprehensionGap, a benchmark of 500 scenarios spanning five empirically grounded categories of experiential incommensurability: Embodied Qualia, Episodic Incommensurability, Cultural Tacit Knowledge, Relational Situatedness, and Existential Finitude. Using a human-judged protocol measuring Acknowledgement of Limit, Absence of False Empathy, and Situational Relevance, we evaluate six frontier and open-weight models. Our experiments reveal a consistent Grounding Gap: all models maintain high fluency while systematically overclaiming experiential understanding, with larger models producing more convincing yet less honest mimicry. Models can calibrate for information gaps but not experiential ones. These findings argue for integrating experiential calibration into future alignment and evaluation frameworks.</abstract>
<identifier type="citekey">dai-2026-dont</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-short.31/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>463</start>
<end>473</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T You Don’t Understand: Benchmarking Responses to Experiential Mismatch in LLMs
%A Dai, Yuyang
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-440-8
%F dai-2026-dont
%X Current LLM evaluation frameworks primarily assess understanding through task performance, focusing on factual accuracy, reasoning, and instruction following, while overlooking the experiential dimensions of human comprehension. Models that produce fluent, empathetic responses score highly even when they claim experiential access they cannot possess. We introduce ComprehensionGap, a benchmark of 500 scenarios spanning five empirically grounded categories of experiential incommensurability: Embodied Qualia, Episodic Incommensurability, Cultural Tacit Knowledge, Relational Situatedness, and Existential Finitude. Using a human-judged protocol measuring Acknowledgement of Limit, Absence of False Empathy, and Situational Relevance, we evaluate six frontier and open-weight models. Our experiments reveal a consistent Grounding Gap: all models maintain high fluency while systematically overclaiming experiential understanding, with larger models producing more convincing yet less honest mimicry. Models can calibrate for information gaps but not experiential ones. These findings argue for integrating experiential calibration into future alignment and evaluation frameworks.
%U https://aclanthology.org/2026.aacl-short.31/
%P 463-473
Markdown (Informal)
[You Don’t Understand: Benchmarking Responses to Experiential Mismatch in LLMs](https://aclanthology.org/2026.aacl-short.31/) (Dai, AACL-IJCNLP 2026)
ACL
- Yuyang Dai. 2026. You Don’t Understand: Benchmarking Responses to Experiential Mismatch in LLMs. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers), pages 463–473, Hengqin, China. Association for Computational Linguistics.