@inproceedings{huang-etal-2026-chemeval2,
title = "{C}hem{E}val2: A Diagnostic Benchmark for Evaluating Tool-Integrated Reasoning in Chemistry",
author = "Huang, Yuqing and
Tang, Chuang and
Liang, Huadong and
Fan, Xianghui and
Wang, Hao and
Xu, Yin and
Xia, Yang and
Jiang, Weiyi and
Liu, De-Guang and
Xu, Feiyang and
Li, Xin and
Wang, Shijin and
Xiao, Mingjun and
Lian, Defu and
Liu, Guiquan and
Chen, Enhong",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.190/",
pages = "4026--4049",
ISBN = "979-8-89176-462-0",
abstract = "Large Language Models (LLMs) are transforming chemical research by enabling specialized reasoning models and tool-augmented agents. Yet existing benchmarks typically evaluate reasoning or tool invocation in isolation, overlooking the dynamic interaction between internal deliberation and external execution. We present ChemEval2, a multimodal benchmark for tool-augmented chemical reasoning, containing 2,410 samples spanning fundamental principles, spectroscopic analysis, and complex reaction understanding. Built from professional textbooks and experimental data through automated generation and rigorous human annotation, ChemEval2 introduces a tool-augmented Chain-of-Thought evaluation framework that embeds tool-use checkpoints at stages where unaided language reasoning becomes unreliable. This design evaluates whether models can identify information gaps, invoke appropriate tools, and integrate tool outputs into reasoning trajectories. Across diverse text and multimodal models, we observe a process{--}outcome inconsistency: models can satisfy intermediate reasoning steps or execute useful tool calls yet still fail at final synthesis. These results provide evidence that coordinating computational tool use with downstream chemical reasoning remains a substantial challenge. We open-source ChemEval2 and an MCP-based suite of specialized chemical tools for reproducible."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="huang-etal-2026-chemeval2">
<titleInfo>
<title>ChemEval2: A Diagnostic Benchmark for Evaluating Tool-Integrated Reasoning in Chemistry</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yuqing</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chuang</namePart>
<namePart type="family">Tang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Huadong</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xianghui</namePart>
<namePart type="family">Fan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hao</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yin</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yang</namePart>
<namePart type="family">Xia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weiyi</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">De-Guang</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Feiyang</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shijin</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mingjun</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Defu</namePart>
<namePart type="family">Lian</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Guiquan</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Enhong</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large Language Models (LLMs) are transforming chemical research by enabling specialized reasoning models and tool-augmented agents. Yet existing benchmarks typically evaluate reasoning or tool invocation in isolation, overlooking the dynamic interaction between internal deliberation and external execution. We present ChemEval2, a multimodal benchmark for tool-augmented chemical reasoning, containing 2,410 samples spanning fundamental principles, spectroscopic analysis, and complex reaction understanding. Built from professional textbooks and experimental data through automated generation and rigorous human annotation, ChemEval2 introduces a tool-augmented Chain-of-Thought evaluation framework that embeds tool-use checkpoints at stages where unaided language reasoning becomes unreliable. This design evaluates whether models can identify information gaps, invoke appropriate tools, and integrate tool outputs into reasoning trajectories. Across diverse text and multimodal models, we observe a process–outcome inconsistency: models can satisfy intermediate reasoning steps or execute useful tool calls yet still fail at final synthesis. These results provide evidence that coordinating computational tool use with downstream chemical reasoning remains a substantial challenge. We open-source ChemEval2 and an MCP-based suite of specialized chemical tools for reproducible.</abstract>
<identifier type="citekey">huang-etal-2026-chemeval2</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.190/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>4026</start>
<end>4049</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T ChemEval2: A Diagnostic Benchmark for Evaluating Tool-Integrated Reasoning in Chemistry
%A Huang, Yuqing
%A Tang, Chuang
%A Liang, Huadong
%A Fan, Xianghui
%A Wang, Hao
%A Xu, Yin
%A Xia, Yang
%A Jiang, Weiyi
%A Liu, De-Guang
%A Xu, Feiyang
%A Li, Xin
%A Wang, Shijin
%A Xiao, Mingjun
%A Lian, Defu
%A Liu, Guiquan
%A Chen, Enhong
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F huang-etal-2026-chemeval2
%X Large Language Models (LLMs) are transforming chemical research by enabling specialized reasoning models and tool-augmented agents. Yet existing benchmarks typically evaluate reasoning or tool invocation in isolation, overlooking the dynamic interaction between internal deliberation and external execution. We present ChemEval2, a multimodal benchmark for tool-augmented chemical reasoning, containing 2,410 samples spanning fundamental principles, spectroscopic analysis, and complex reaction understanding. Built from professional textbooks and experimental data through automated generation and rigorous human annotation, ChemEval2 introduces a tool-augmented Chain-of-Thought evaluation framework that embeds tool-use checkpoints at stages where unaided language reasoning becomes unreliable. This design evaluates whether models can identify information gaps, invoke appropriate tools, and integrate tool outputs into reasoning trajectories. Across diverse text and multimodal models, we observe a process–outcome inconsistency: models can satisfy intermediate reasoning steps or execute useful tool calls yet still fail at final synthesis. These results provide evidence that coordinating computational tool use with downstream chemical reasoning remains a substantial challenge. We open-source ChemEval2 and an MCP-based suite of specialized chemical tools for reproducible.
%U https://aclanthology.org/2026.findings-emnlp.190/
%P 4026-4049
Markdown (Informal)
[ChemEval2: A Diagnostic Benchmark for Evaluating Tool-Integrated Reasoning in Chemistry](https://aclanthology.org/2026.findings-emnlp.190/) (Huang et al., Findings 2026)
ACL
- Yuqing Huang, Chuang Tang, Huadong Liang, Xianghui Fan, Hao Wang, Yin Xu, Yang Xia, Weiyi Jiang, De-Guang Liu, Feiyang Xu, Xin Li, Shijin Wang, Mingjun Xiao, Defu Lian, Guiquan Liu, and Enhong Chen. 2026. ChemEval2: A Diagnostic Benchmark for Evaluating Tool-Integrated Reasoning in Chemistry. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 4026–4049, Budapest, Hungary. Association for Computational Linguistics.