@inproceedings{ishikawa-etal-2026-hydra,
title = "{HYDRA}: Evaluating Logical Thinking Abilities of {LLM}s through Formulating Hypotheses and Verification Methods",
author = "Ishikawa, Shin-nosuke and
Ohba, Hirotsugu and
Todo, Masato and
Yoshino, Atsushi and
Daimon, Masahiro",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.206/",
pages = "4090--4105",
ISBN = "979-8-89176-444-6",
abstract = "Logical consistency in the outputs of large language models (LLMs) is essential for extending their applications beyond single-task assistants, including their use in AI agent systems. Although logical reasoning capabilities are often evaluated through tasks such as mathematics and coding, where it is feasible to assess the correctness of outputs in a well-defined manner, real-world problem solving requires inductive and abductive reasoning, with an emphasis on the ability to formulate and verify hypotheses. When evaluating the hypothesis formulation and verification capabilities of LLMs, the major challenge lies in preparing a variety of tasks with appropriate difficulty, avoiding those that are either too easy or unsolvable. We introduce an automated framework called HYpothesis-Driven Reasoning Assessment (HYDRA), focusing on hypothesis formulation and verification for real-world problem solving with logical reasoning. In HYDRA, all components, including situation generation, hypothesis, and evaluation, are fully performed by LLMs. HYDRA{'}s systematically constructed generation and evaluation strategies enable assessment using realistic, non-trivial situations that offer both diversity and clear criteria. We evaluated six state-of-the-art LLMs using HYDRA and found that their success rates ranged from 74.3{\%} to 90.3{\%}, indicating that even the best-performing model among those evaluated produced rubric-invalid hypothesis{--}verification pairs in nearly 10{\%} of cases."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ishikawa-etal-2026-hydra">
<titleInfo>
<title>HYDRA: Evaluating Logical Thinking Abilities of LLMs through Formulating Hypotheses and Verification Methods</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shin-nosuke</namePart>
<namePart type="family">Ishikawa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hirotsugu</namePart>
<namePart type="family">Ohba</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Masato</namePart>
<namePart type="family">Todo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Atsushi</namePart>
<namePart type="family">Yoshino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Masahiro</namePart>
<namePart type="family">Daimon</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>Logical consistency in the outputs of large language models (LLMs) is essential for extending their applications beyond single-task assistants, including their use in AI agent systems. Although logical reasoning capabilities are often evaluated through tasks such as mathematics and coding, where it is feasible to assess the correctness of outputs in a well-defined manner, real-world problem solving requires inductive and abductive reasoning, with an emphasis on the ability to formulate and verify hypotheses. When evaluating the hypothesis formulation and verification capabilities of LLMs, the major challenge lies in preparing a variety of tasks with appropriate difficulty, avoiding those that are either too easy or unsolvable. We introduce an automated framework called HYpothesis-Driven Reasoning Assessment (HYDRA), focusing on hypothesis formulation and verification for real-world problem solving with logical reasoning. In HYDRA, all components, including situation generation, hypothesis, and evaluation, are fully performed by LLMs. HYDRA’s systematically constructed generation and evaluation strategies enable assessment using realistic, non-trivial situations that offer both diversity and clear criteria. We evaluated six state-of-the-art LLMs using HYDRA and found that their success rates ranged from 74.3% to 90.3%, indicating that even the best-performing model among those evaluated produced rubric-invalid hypothesis–verification pairs in nearly 10% of cases.</abstract>
<identifier type="citekey">ishikawa-etal-2026-hydra</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.206/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>4090</start>
<end>4105</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T HYDRA: Evaluating Logical Thinking Abilities of LLMs through Formulating Hypotheses and Verification Methods
%A Ishikawa, Shin-nosuke
%A Ohba, Hirotsugu
%A Todo, Masato
%A Yoshino, Atsushi
%A Daimon, Masahiro
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F ishikawa-etal-2026-hydra
%X Logical consistency in the outputs of large language models (LLMs) is essential for extending their applications beyond single-task assistants, including their use in AI agent systems. Although logical reasoning capabilities are often evaluated through tasks such as mathematics and coding, where it is feasible to assess the correctness of outputs in a well-defined manner, real-world problem solving requires inductive and abductive reasoning, with an emphasis on the ability to formulate and verify hypotheses. When evaluating the hypothesis formulation and verification capabilities of LLMs, the major challenge lies in preparing a variety of tasks with appropriate difficulty, avoiding those that are either too easy or unsolvable. We introduce an automated framework called HYpothesis-Driven Reasoning Assessment (HYDRA), focusing on hypothesis formulation and verification for real-world problem solving with logical reasoning. In HYDRA, all components, including situation generation, hypothesis, and evaluation, are fully performed by LLMs. HYDRA’s systematically constructed generation and evaluation strategies enable assessment using realistic, non-trivial situations that offer both diversity and clear criteria. We evaluated six state-of-the-art LLMs using HYDRA and found that their success rates ranged from 74.3% to 90.3%, indicating that even the best-performing model among those evaluated produced rubric-invalid hypothesis–verification pairs in nearly 10% of cases.
%U https://aclanthology.org/2026.findings-aacl.206/
%P 4090-4105
Markdown (Informal)
[HYDRA: Evaluating Logical Thinking Abilities of LLMs through Formulating Hypotheses and Verification Methods](https://aclanthology.org/2026.findings-aacl.206/) (Ishikawa et al., Findings 2026)
ACL