@inproceedings{garg-etal-2026-towards,
title = "Towards {AI}-Assisted Research Writing: Benchmarking {LLM}s for {AI}/{ML} Introduction Generation",
author = "Garg, Krishna and
Shaik, Firoz and
Bandyopadhyay, Sambaran and
Caragea, Cornelia",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1288/",
pages = "29341--29365",
ISBN = "979-8-89176-459-0",
abstract = "As researchers increasingly adopt LLMs as writing assistants, generating high-quality research paper introductions remains both challenging and essential. In this work, we benchmark Scientific Introduction Generation (SciIG), evaluating how well LLMs can produce coherent introductions from titles, abstracts, and related works in the machine learning domain. Curating new datasets from NAACL 2025 and ICLR 2025 papers, we assess five state-of-the-art models, including both open-source (DeepSeek-v3, Gemma-3-12B, LLaMA-4 Maverick, MistralAI Small 3.1) and closed-source GPT-4o systems across multiple dimensions: lexical overlap, semantic similarity, content coverage, faithfulness, consistency, citation correctness, and narrative quality. Our comprehensive framework combines automated metrics with LLM-as-a-judge evaluations. Results demonstrate LLaMA-4 Maverick{'}s superior performance on most metrics, particularly in lexical overlap, content coverage, citation quality, and perplexity. Moreover, three-shot prompting outperforms fewer-shot approaches more than 50{\%} of the time on formula-based automated metrics and performs consistently better on all human evaluation metrics. These findings provide practical insights into developing effective research writing assistants and set realistic expectations for LLM-assisted academic writing. To foster reproducibility and future research, we publicly release all code and datasets.[Project page, code, and datasets: {\ensuremath{<}}https://kgarg8.github.io/emnlp26-sciig/{\ensuremath{>}}]"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="garg-etal-2026-towards">
<titleInfo>
<title>Towards AI-Assisted Research Writing: Benchmarking LLMs for AI/ML Introduction Generation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Krishna</namePart>
<namePart type="family">Garg</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Firoz</namePart>
<namePart type="family">Shaik</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sambaran</namePart>
<namePart type="family">Bandyopadhyay</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cornelia</namePart>
<namePart type="family">Caragea</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>As researchers increasingly adopt LLMs as writing assistants, generating high-quality research paper introductions remains both challenging and essential. In this work, we benchmark Scientific Introduction Generation (SciIG), evaluating how well LLMs can produce coherent introductions from titles, abstracts, and related works in the machine learning domain. Curating new datasets from NAACL 2025 and ICLR 2025 papers, we assess five state-of-the-art models, including both open-source (DeepSeek-v3, Gemma-3-12B, LLaMA-4 Maverick, MistralAI Small 3.1) and closed-source GPT-4o systems across multiple dimensions: lexical overlap, semantic similarity, content coverage, faithfulness, consistency, citation correctness, and narrative quality. Our comprehensive framework combines automated metrics with LLM-as-a-judge evaluations. Results demonstrate LLaMA-4 Maverick’s superior performance on most metrics, particularly in lexical overlap, content coverage, citation quality, and perplexity. Moreover, three-shot prompting outperforms fewer-shot approaches more than 50% of the time on formula-based automated metrics and performs consistently better on all human evaluation metrics. These findings provide practical insights into developing effective research writing assistants and set realistic expectations for LLM-assisted academic writing. To foster reproducibility and future research, we publicly release all code and datasets.[Project page, code, and datasets: \ensuremath<https://kgarg8.github.io/emnlp26-sciig/\ensuremath>]</abstract>
<identifier type="citekey">garg-etal-2026-towards</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1288/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>29341</start>
<end>29365</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Towards AI-Assisted Research Writing: Benchmarking LLMs for AI/ML Introduction Generation
%A Garg, Krishna
%A Shaik, Firoz
%A Bandyopadhyay, Sambaran
%A Caragea, Cornelia
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F garg-etal-2026-towards
%X As researchers increasingly adopt LLMs as writing assistants, generating high-quality research paper introductions remains both challenging and essential. In this work, we benchmark Scientific Introduction Generation (SciIG), evaluating how well LLMs can produce coherent introductions from titles, abstracts, and related works in the machine learning domain. Curating new datasets from NAACL 2025 and ICLR 2025 papers, we assess five state-of-the-art models, including both open-source (DeepSeek-v3, Gemma-3-12B, LLaMA-4 Maverick, MistralAI Small 3.1) and closed-source GPT-4o systems across multiple dimensions: lexical overlap, semantic similarity, content coverage, faithfulness, consistency, citation correctness, and narrative quality. Our comprehensive framework combines automated metrics with LLM-as-a-judge evaluations. Results demonstrate LLaMA-4 Maverick’s superior performance on most metrics, particularly in lexical overlap, content coverage, citation quality, and perplexity. Moreover, three-shot prompting outperforms fewer-shot approaches more than 50% of the time on formula-based automated metrics and performs consistently better on all human evaluation metrics. These findings provide practical insights into developing effective research writing assistants and set realistic expectations for LLM-assisted academic writing. To foster reproducibility and future research, we publicly release all code and datasets.[Project page, code, and datasets: \ensuremath<https://kgarg8.github.io/emnlp26-sciig/\ensuremath>]
%U https://aclanthology.org/2026.emnlp-main.1288/
%P 29341-29365
Markdown (Informal)
[Towards AI-Assisted Research Writing: Benchmarking LLMs for AI/ML Introduction Generation](https://aclanthology.org/2026.emnlp-main.1288/) (Garg et al., EMNLP 2026)
ACL