@inproceedings{chen-etal-2026-truly,
title = "``Do You Truly Love Me?'' Benchmarking {LLM} Capability on Hierarchical Pragmatic Tactic Conversations",
author = "Chen, Peter Q. and
Wu, Pangjing and
Jiang, Zhuohang and
Li, Xiaodong and
Ng, Peter H. F.",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.191/",
pages = "4261--4281",
ISBN = "979-8-89176-439-2",
abstract = "Pragmatic reasoning, including inferring intent, manipulation, and hidden meaning in dialogue, is essential for trustworthy language understanding yet remains underexplored in current large language models (LLMs). We introduce \textbf{HIPO}, a new benchmark designed to assess \textbf{\underline{Hi}}erarchical \textbf{\underline{P}}ragmatic Tactic in multi-turn C\textbf{\underline{o}}nversations. Inspired by linguistic pragmatics, HIPO operationalizes a hierarchical annotation schema for multi-turn conversations and labels each utterance along one dialogue-level goal and three utterance-level dimensions: communicative intention (\textit{why}), veracity strategy (\textit{how}), and illocutionary act (\textit{what}). To ensure reliable supervision, we design a structured generation pipeline to generate high-quality synthetic dialogues. The benchmark comprises over 4,088 benchmarking utterances with 6,350 contextual utterances across 1,131 dialogues drawn from 31 real-world-inspired scenarios. Benchmarking 22 state-of-the-art LLMs reveals a striking gap: while models achieve higher performance at recognizing surface speech acts(83.3{\%} accuracy), they perform poorly on detecting veracity strategies(32.2{\%}) and speaker intentions(45.1{\%}). These results suggest current LLMs remain challenged by the pragmatic inferences operationalized in HIPO, particularly those involving information manipulation and intended effects beyond literal content. We further show that supervised fine-tuning on additional synthetic data generated by our pipeline can improve performance on HIPO, suggesting a practical path for targeted capability improvement within this benchmark setting."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chen-etal-2026-truly">
<titleInfo>
<title>“Do You Truly Love Me?” Benchmarking LLM Capability on Hierarchical Pragmatic Tactic Conversations</title>
</titleInfo>
<name type="personal">
<namePart type="given">Peter</namePart>
<namePart type="given">Q</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pangjing</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhuohang</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaodong</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peter</namePart>
<namePart type="given">H</namePart>
<namePart type="given">F</namePart>
<namePart type="family">Ng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Pragmatic reasoning, including inferring intent, manipulation, and hidden meaning in dialogue, is essential for trustworthy language understanding yet remains underexplored in current large language models (LLMs). We introduce HIPO, a new benchmark designed to assess \underlineHierarchical \underlinePragmatic Tactic in multi-turn C\underlineonversations. Inspired by linguistic pragmatics, HIPO operationalizes a hierarchical annotation schema for multi-turn conversations and labels each utterance along one dialogue-level goal and three utterance-level dimensions: communicative intention (why), veracity strategy (how), and illocutionary act (what). To ensure reliable supervision, we design a structured generation pipeline to generate high-quality synthetic dialogues. The benchmark comprises over 4,088 benchmarking utterances with 6,350 contextual utterances across 1,131 dialogues drawn from 31 real-world-inspired scenarios. Benchmarking 22 state-of-the-art LLMs reveals a striking gap: while models achieve higher performance at recognizing surface speech acts(83.3% accuracy), they perform poorly on detecting veracity strategies(32.2%) and speaker intentions(45.1%). These results suggest current LLMs remain challenged by the pragmatic inferences operationalized in HIPO, particularly those involving information manipulation and intended effects beyond literal content. We further show that supervised fine-tuning on additional synthetic data generated by our pipeline can improve performance on HIPO, suggesting a practical path for targeted capability improvement within this benchmark setting.</abstract>
<identifier type="citekey">chen-etal-2026-truly</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.191/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>4261</start>
<end>4281</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T “Do You Truly Love Me?” Benchmarking LLM Capability on Hierarchical Pragmatic Tactic Conversations
%A Chen, Peter Q.
%A Wu, Pangjing
%A Jiang, Zhuohang
%A Li, Xiaodong
%A Ng, Peter H. F.
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F chen-etal-2026-truly
%X Pragmatic reasoning, including inferring intent, manipulation, and hidden meaning in dialogue, is essential for trustworthy language understanding yet remains underexplored in current large language models (LLMs). We introduce HIPO, a new benchmark designed to assess \underlineHierarchical \underlinePragmatic Tactic in multi-turn C\underlineonversations. Inspired by linguistic pragmatics, HIPO operationalizes a hierarchical annotation schema for multi-turn conversations and labels each utterance along one dialogue-level goal and three utterance-level dimensions: communicative intention (why), veracity strategy (how), and illocutionary act (what). To ensure reliable supervision, we design a structured generation pipeline to generate high-quality synthetic dialogues. The benchmark comprises over 4,088 benchmarking utterances with 6,350 contextual utterances across 1,131 dialogues drawn from 31 real-world-inspired scenarios. Benchmarking 22 state-of-the-art LLMs reveals a striking gap: while models achieve higher performance at recognizing surface speech acts(83.3% accuracy), they perform poorly on detecting veracity strategies(32.2%) and speaker intentions(45.1%). These results suggest current LLMs remain challenged by the pragmatic inferences operationalized in HIPO, particularly those involving information manipulation and intended effects beyond literal content. We further show that supervised fine-tuning on additional synthetic data generated by our pipeline can improve performance on HIPO, suggesting a practical path for targeted capability improvement within this benchmark setting.
%U https://aclanthology.org/2026.aacl-long.191/
%P 4261-4281
Markdown (Informal)
[“Do You Truly Love Me?” Benchmarking LLM Capability on Hierarchical Pragmatic Tactic Conversations](https://aclanthology.org/2026.aacl-long.191/) (Chen et al., AACL-IJCNLP 2026)
ACL