@inproceedings{cong-2026-judging,
title = "Judging the judge: Can {LLM}s track attitude-holders in predicates of personal taste?",
author = "Cong, Yan",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.19/",
pages = "342--349",
ISBN = "979-8-89176-444-6",
abstract = "When LLMs annotate subjective content such as predicates of personal taste (PPTs), e.g., ``Eliza tasted the cookie. It tasted delicious'', a consequential but underexplored question is whose perspective they adopt: the narrator{'}s (default) or Eliza{'}s (shift toward the character)? Humans tend to attribute ``delicious'' to Eliza because tasting implies first-hand experience. We introduce GRANI (Graded Response Analysis for Narrative Implications), a protocol for evaluating LLMs' graded understanding of such phenomenon. Applying GRANI to PPTs, we test whether text-trained LLMs reproduce human perspective shifts across sensory modalities in English and Chinese. We found that instruction-tuned models partially capture the human pattern, in which taste yields the strongest shift toward the character, vision the weakest, and smell an intermediate effect, while reasoning-distilled models are less consistent. Moreover, some models encode modality distinctions internally even when their behavioral responses do not. These findings suggest that subjectivity inference can emerge from text exposure alone, while highlighting a reliability concern: LLMs may not consistently adopt the intended perspective-holder when evaluating subjective content."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="cong-2026-judging">
<titleInfo>
<title>Judging the judge: Can LLMs track attitude-holders in predicates of personal taste?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yan</namePart>
<namePart type="family">Cong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>When LLMs annotate subjective content such as predicates of personal taste (PPTs), e.g., “Eliza tasted the cookie. It tasted delicious”, a consequential but underexplored question is whose perspective they adopt: the narrator’s (default) or Eliza’s (shift toward the character)? Humans tend to attribute “delicious” to Eliza because tasting implies first-hand experience. We introduce GRANI (Graded Response Analysis for Narrative Implications), a protocol for evaluating LLMs’ graded understanding of such phenomenon. Applying GRANI to PPTs, we test whether text-trained LLMs reproduce human perspective shifts across sensory modalities in English and Chinese. We found that instruction-tuned models partially capture the human pattern, in which taste yields the strongest shift toward the character, vision the weakest, and smell an intermediate effect, while reasoning-distilled models are less consistent. Moreover, some models encode modality distinctions internally even when their behavioral responses do not. These findings suggest that subjectivity inference can emerge from text exposure alone, while highlighting a reliability concern: LLMs may not consistently adopt the intended perspective-holder when evaluating subjective content.</abstract>
<identifier type="citekey">cong-2026-judging</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.19/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>342</start>
<end>349</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Judging the judge: Can LLMs track attitude-holders in predicates of personal taste?
%A Cong, Yan
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F cong-2026-judging
%X When LLMs annotate subjective content such as predicates of personal taste (PPTs), e.g., “Eliza tasted the cookie. It tasted delicious”, a consequential but underexplored question is whose perspective they adopt: the narrator’s (default) or Eliza’s (shift toward the character)? Humans tend to attribute “delicious” to Eliza because tasting implies first-hand experience. We introduce GRANI (Graded Response Analysis for Narrative Implications), a protocol for evaluating LLMs’ graded understanding of such phenomenon. Applying GRANI to PPTs, we test whether text-trained LLMs reproduce human perspective shifts across sensory modalities in English and Chinese. We found that instruction-tuned models partially capture the human pattern, in which taste yields the strongest shift toward the character, vision the weakest, and smell an intermediate effect, while reasoning-distilled models are less consistent. Moreover, some models encode modality distinctions internally even when their behavioral responses do not. These findings suggest that subjectivity inference can emerge from text exposure alone, while highlighting a reliability concern: LLMs may not consistently adopt the intended perspective-holder when evaluating subjective content.
%U https://aclanthology.org/2026.findings-aacl.19/
%P 342-349
Markdown (Informal)
[Judging the judge: Can LLMs track attitude-holders in predicates of personal taste?](https://aclanthology.org/2026.findings-aacl.19/) (Cong, Findings 2026)
ACL