@inproceedings{ko-etal-2026-llms,
title = "Can {LLM}s Reliably Identify Causal Conclusions in Scientific Literature? A Multidisciplinary Evaluation",
author = "Ko, Young Soo and
Sharief, Nadia and
Lu, Xuan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.597/",
pages = "12592--12609",
ISBN = "979-8-89176-462-0",
abstract = "Large language models (LLMs) hold promise for automating causal knowledge extraction, a fundamental component of scientific discovery, yet their ability to reliably identify causal claims in scientific literature remains poorly understood. Using 1,000 manually annotated abstracts spanning 20 disciplines, we evaluate eight frontier LLMs under four prompting strategies on a causal conclusion identification task. More elaborate prompting strategies do not necessarily improve performance, while open-weight models achieve competitive performance relative to proprietary models. LLMs systematically produce false-positive causal predictions on non-causal abstracts, with GPT-5 showing the most extreme bias, and Few-Shot prompting provides the most consistent calibration benefit. Performance varies substantially across disciplines with stronger and more stable results in biomedical fields. Our findings provide practical insights for using LLMs in scientific causal analysis and highlight calibration and domain variation as key challenges for advancing causal reasoning in scholarly text."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ko-etal-2026-llms">
<titleInfo>
<title>Can LLMs Reliably Identify Causal Conclusions in Scientific Literature? A Multidisciplinary Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Young</namePart>
<namePart type="given">Soo</namePart>
<namePart type="family">Ko</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nadia</namePart>
<namePart type="family">Sharief</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xuan</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large language models (LLMs) hold promise for automating causal knowledge extraction, a fundamental component of scientific discovery, yet their ability to reliably identify causal claims in scientific literature remains poorly understood. Using 1,000 manually annotated abstracts spanning 20 disciplines, we evaluate eight frontier LLMs under four prompting strategies on a causal conclusion identification task. More elaborate prompting strategies do not necessarily improve performance, while open-weight models achieve competitive performance relative to proprietary models. LLMs systematically produce false-positive causal predictions on non-causal abstracts, with GPT-5 showing the most extreme bias, and Few-Shot prompting provides the most consistent calibration benefit. Performance varies substantially across disciplines with stronger and more stable results in biomedical fields. Our findings provide practical insights for using LLMs in scientific causal analysis and highlight calibration and domain variation as key challenges for advancing causal reasoning in scholarly text.</abstract>
<identifier type="citekey">ko-etal-2026-llms</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.597/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>12592</start>
<end>12609</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Can LLMs Reliably Identify Causal Conclusions in Scientific Literature? A Multidisciplinary Evaluation
%A Ko, Young Soo
%A Sharief, Nadia
%A Lu, Xuan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F ko-etal-2026-llms
%X Large language models (LLMs) hold promise for automating causal knowledge extraction, a fundamental component of scientific discovery, yet their ability to reliably identify causal claims in scientific literature remains poorly understood. Using 1,000 manually annotated abstracts spanning 20 disciplines, we evaluate eight frontier LLMs under four prompting strategies on a causal conclusion identification task. More elaborate prompting strategies do not necessarily improve performance, while open-weight models achieve competitive performance relative to proprietary models. LLMs systematically produce false-positive causal predictions on non-causal abstracts, with GPT-5 showing the most extreme bias, and Few-Shot prompting provides the most consistent calibration benefit. Performance varies substantially across disciplines with stronger and more stable results in biomedical fields. Our findings provide practical insights for using LLMs in scientific causal analysis and highlight calibration and domain variation as key challenges for advancing causal reasoning in scholarly text.
%U https://aclanthology.org/2026.findings-emnlp.597/
%P 12592-12609
Markdown (Informal)
[Can LLMs Reliably Identify Causal Conclusions in Scientific Literature? A Multidisciplinary Evaluation](https://aclanthology.org/2026.findings-emnlp.597/) (Ko et al., Findings 2026)
ACL