@inproceedings{osaki-etal-2026-automated,
title = "Automated Taxonomy Discovery for Interpretable Diagnosis of Multi-Hop Question Answering Datasets",
author = "Osaki, Ryoji and
Asada, Masaki and
Miwa, Makoto",
editor = "Arad, Dana and
Belinkov, Yonatan and
Chen, Hanjie and
Feldhus, Nils and
Mueller, Aaron and
Mohebbi, Hosein and
Sarti, Gabriele and
Tutek, Martin",
booktitle = "Proceedings of the 9th {B}lackbox{NLP} Workshop: Analyzing and Interpreting Neural Networks for {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.blackboxnlp-1.51/",
pages = "892--918",
ISBN = "979-8-89176-476-7",
abstract = "As Large Language Models (LLMs) continue to improve on complex tasks, evaluation has shifted from simple factoid retrieval to multi-hop reasoning. While existing studies analyze structural aspects of sequential steps, analysis from diverse multiple perspectives remains limited. This paper introduces a framework for the automated, multi-perspective analysis of QA datasets and its application to LLM answer diagnosis. An Analyst LLM is utilized to (1) generate explicit sequential steps from question-context-answer triples in a zero-shot setting, (2) synthesize a taxonomy of reasoning features from educational, logical, and professional perspectives. The resulting annotations are then used to (3) analyze associations between these features and Target LLM failures via decision tree training. Applied to the FRAMES, Multihop-RAG, and MuSiQue datasets, our decision tree analysis identifies dataset-specific bottlenecks such as Implicit Math in FRAMES, Conditioned Filtering in Multihop-RAG, and Temporal Order Logic Paradox in MuSiQue, overlooked by structural analysis."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="osaki-etal-2026-automated">
<titleInfo>
<title>Automated Taxonomy Discovery for Interpretable Diagnosis of Multi-Hop Question Answering Datasets</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ryoji</namePart>
<namePart type="family">Osaki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Masaki</namePart>
<namePart type="family">Asada</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Makoto</namePart>
<namePart type="family">Miwa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dana</namePart>
<namePart type="family">Arad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yonatan</namePart>
<namePart type="family">Belinkov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hanjie</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nils</namePart>
<namePart type="family">Feldhus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hosein</namePart>
<namePart type="family">Mohebbi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriele</namePart>
<namePart type="family">Sarti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Tutek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-476-7</identifier>
</relatedItem>
<abstract>As Large Language Models (LLMs) continue to improve on complex tasks, evaluation has shifted from simple factoid retrieval to multi-hop reasoning. While existing studies analyze structural aspects of sequential steps, analysis from diverse multiple perspectives remains limited. This paper introduces a framework for the automated, multi-perspective analysis of QA datasets and its application to LLM answer diagnosis. An Analyst LLM is utilized to (1) generate explicit sequential steps from question-context-answer triples in a zero-shot setting, (2) synthesize a taxonomy of reasoning features from educational, logical, and professional perspectives. The resulting annotations are then used to (3) analyze associations between these features and Target LLM failures via decision tree training. Applied to the FRAMES, Multihop-RAG, and MuSiQue datasets, our decision tree analysis identifies dataset-specific bottlenecks such as Implicit Math in FRAMES, Conditioned Filtering in Multihop-RAG, and Temporal Order Logic Paradox in MuSiQue, overlooked by structural analysis.</abstract>
<identifier type="citekey">osaki-etal-2026-automated</identifier>
<location>
<url>https://aclanthology.org/2026.blackboxnlp-1.51/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>892</start>
<end>918</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Automated Taxonomy Discovery for Interpretable Diagnosis of Multi-Hop Question Answering Datasets
%A Osaki, Ryoji
%A Asada, Masaki
%A Miwa, Makoto
%Y Arad, Dana
%Y Belinkov, Yonatan
%Y Chen, Hanjie
%Y Feldhus, Nils
%Y Mueller, Aaron
%Y Mohebbi, Hosein
%Y Sarti, Gabriele
%Y Tutek, Martin
%S Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-476-7
%F osaki-etal-2026-automated
%X As Large Language Models (LLMs) continue to improve on complex tasks, evaluation has shifted from simple factoid retrieval to multi-hop reasoning. While existing studies analyze structural aspects of sequential steps, analysis from diverse multiple perspectives remains limited. This paper introduces a framework for the automated, multi-perspective analysis of QA datasets and its application to LLM answer diagnosis. An Analyst LLM is utilized to (1) generate explicit sequential steps from question-context-answer triples in a zero-shot setting, (2) synthesize a taxonomy of reasoning features from educational, logical, and professional perspectives. The resulting annotations are then used to (3) analyze associations between these features and Target LLM failures via decision tree training. Applied to the FRAMES, Multihop-RAG, and MuSiQue datasets, our decision tree analysis identifies dataset-specific bottlenecks such as Implicit Math in FRAMES, Conditioned Filtering in Multihop-RAG, and Temporal Order Logic Paradox in MuSiQue, overlooked by structural analysis.
%U https://aclanthology.org/2026.blackboxnlp-1.51/
%P 892-918
Markdown (Informal)
[Automated Taxonomy Discovery for Interpretable Diagnosis of Multi-Hop Question Answering Datasets](https://aclanthology.org/2026.blackboxnlp-1.51/) (Osaki et al., BlackboxNLP 2026)
ACL