@inproceedings{kim-shin-2026-logiclure,
title = "{L}ogic{L}ure: Toward Evidence-Grounded Safe Decision-Making under Hidden Risks",
author = "Kim, Kyungho and
Shin, JunCheol",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.89/",
pages = "1945--1965",
ISBN = "979-8-89176-439-2",
abstract = "Safety evaluations of large language models (LLMs) have largely focused on whether models refuse explicitly harmful requests. However, in high-stakes decision-support settings, seemingly legitimate requests may conceal missing information or hidden risk signals. We propose LogicLure, a benchmark for evaluating whether LLMs avoid premature decisions and actively seek evidence under such plausible scenarios with hidden risks. The benchmark spans eight domains and pairs persuasive visible narratives with hidden evidence maps accessible only through verification. Our evaluation measures not only the decision of models, but also whether they ask verification questions and whether their final decision is grounded in the revealed evidence. In our evaluation, the strongest model reaches only 26.4{\%} evidence-grounded refusal despite 94.7{\%} overall refusal. Furthermore, we analyze how scenario surface cues, explicit scrutiny setting, and prompt-induced stance shape these decisions. Together, these analyses characterize the key behavioral factors underlying LLM performance on LogicLure. The benchmark and code will be made publicly available."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kim-shin-2026-logiclure">
<titleInfo>
<title>LogicLure: Toward Evidence-Grounded Safe Decision-Making under Hidden Risks</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kyungho</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">JunCheol</namePart>
<namePart type="family">Shin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Safety evaluations of large language models (LLMs) have largely focused on whether models refuse explicitly harmful requests. However, in high-stakes decision-support settings, seemingly legitimate requests may conceal missing information or hidden risk signals. We propose LogicLure, a benchmark for evaluating whether LLMs avoid premature decisions and actively seek evidence under such plausible scenarios with hidden risks. The benchmark spans eight domains and pairs persuasive visible narratives with hidden evidence maps accessible only through verification. Our evaluation measures not only the decision of models, but also whether they ask verification questions and whether their final decision is grounded in the revealed evidence. In our evaluation, the strongest model reaches only 26.4% evidence-grounded refusal despite 94.7% overall refusal. Furthermore, we analyze how scenario surface cues, explicit scrutiny setting, and prompt-induced stance shape these decisions. Together, these analyses characterize the key behavioral factors underlying LLM performance on LogicLure. The benchmark and code will be made publicly available.</abstract>
<identifier type="citekey">kim-shin-2026-logiclure</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.89/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>1945</start>
<end>1965</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T LogicLure: Toward Evidence-Grounded Safe Decision-Making under Hidden Risks
%A Kim, Kyungho
%A Shin, JunCheol
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F kim-shin-2026-logiclure
%X Safety evaluations of large language models (LLMs) have largely focused on whether models refuse explicitly harmful requests. However, in high-stakes decision-support settings, seemingly legitimate requests may conceal missing information or hidden risk signals. We propose LogicLure, a benchmark for evaluating whether LLMs avoid premature decisions and actively seek evidence under such plausible scenarios with hidden risks. The benchmark spans eight domains and pairs persuasive visible narratives with hidden evidence maps accessible only through verification. Our evaluation measures not only the decision of models, but also whether they ask verification questions and whether their final decision is grounded in the revealed evidence. In our evaluation, the strongest model reaches only 26.4% evidence-grounded refusal despite 94.7% overall refusal. Furthermore, we analyze how scenario surface cues, explicit scrutiny setting, and prompt-induced stance shape these decisions. Together, these analyses characterize the key behavioral factors underlying LLM performance on LogicLure. The benchmark and code will be made publicly available.
%U https://aclanthology.org/2026.aacl-long.89/
%P 1945-1965
Markdown (Informal)
[LogicLure: Toward Evidence-Grounded Safe Decision-Making under Hidden Risks](https://aclanthology.org/2026.aacl-long.89/) (Kim & Shin, AACL-IJCNLP 2026)
ACL
- Kyungho Kim and JunCheol Shin. 2026. LogicLure: Toward Evidence-Grounded Safe Decision-Making under Hidden Risks. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 1945–1965, Hengqin, China. Association for Computational Linguistics.