@inproceedings{wang-etal-2026-deploybench,
title = "{D}eploy{B}ench : Benchmarking {LLM} Agents for Research Artifact Deployment",
author = "Wang, Yuanli and
Qian, Yaoyao and
Zhang, Yue and
Zhou, Hanhan and
Huang, Jindan and
Fu, Tianfu and
Mang, Qiuyang and
Mao, Huanzhi and
Chai, Wenhao and
Fan, Wendong and
Jing, Liqiang",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.707/",
pages = "14848--14866",
ISBN = "979-8-89176-462-0",
abstract = "LLM agents have made rapid progress on software engineering and ML research tasks, but these advances often assume access to a working runnable environment. For research artifacts released alongside published papers, setting up such an environment from a fresh machine remains a major bottleneck. Existing environment setup benchmarks do not cover the full scope of research-artifact deployment, which involves multi-language toolchains, system-level dependencies beyond containers (e.g., GPU/CUDA and kernel configurations), and legacy artifact compatibility. We introduce DeployBench, a multi-domain benchmark of 51 research-artifact deployment tasks spanning AI/ML, computer systems, and scientific computing, covering all these dimensions. Each task is verified by a hidden pipeline that executes the paper{'}s designated experiment and checks its outputs. Evaluating five state-of-the-art LLMs with OpenHands yields pass-rates from 7.8{\%}-51.0{\%}. Failures are dominated by a completion-judgment problem: 102 of 181 are agent-terminated self-stops, where the agent{'}s pre-finish checks validate a different or weaker target than the paper-specific task requires. DeployBench highlights the gap between current agents and autonomous deployment, and offers a realistic testbed for scientific research agents."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-deploybench">
<titleInfo>
<title>DeployBench : Benchmarking LLM Agents for Research Artifact Deployment</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yuanli</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yaoyao</namePart>
<namePart type="family">Qian</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hanhan</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jindan</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianfu</namePart>
<namePart type="family">Fu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qiuyang</namePart>
<namePart type="family">Mang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Huanzhi</namePart>
<namePart type="family">Mao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenhao</namePart>
<namePart type="family">Chai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wendong</namePart>
<namePart type="family">Fan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Liqiang</namePart>
<namePart type="family">Jing</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>LLM agents have made rapid progress on software engineering and ML research tasks, but these advances often assume access to a working runnable environment. For research artifacts released alongside published papers, setting up such an environment from a fresh machine remains a major bottleneck. Existing environment setup benchmarks do not cover the full scope of research-artifact deployment, which involves multi-language toolchains, system-level dependencies beyond containers (e.g., GPU/CUDA and kernel configurations), and legacy artifact compatibility. We introduce DeployBench, a multi-domain benchmark of 51 research-artifact deployment tasks spanning AI/ML, computer systems, and scientific computing, covering all these dimensions. Each task is verified by a hidden pipeline that executes the paper’s designated experiment and checks its outputs. Evaluating five state-of-the-art LLMs with OpenHands yields pass-rates from 7.8%-51.0%. Failures are dominated by a completion-judgment problem: 102 of 181 are agent-terminated self-stops, where the agent’s pre-finish checks validate a different or weaker target than the paper-specific task requires. DeployBench highlights the gap between current agents and autonomous deployment, and offers a realistic testbed for scientific research agents.</abstract>
<identifier type="citekey">wang-etal-2026-deploybench</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.707/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>14848</start>
<end>14866</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T DeployBench : Benchmarking LLM Agents for Research Artifact Deployment
%A Wang, Yuanli
%A Qian, Yaoyao
%A Zhang, Yue
%A Zhou, Hanhan
%A Huang, Jindan
%A Fu, Tianfu
%A Mang, Qiuyang
%A Mao, Huanzhi
%A Chai, Wenhao
%A Fan, Wendong
%A Jing, Liqiang
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F wang-etal-2026-deploybench
%X LLM agents have made rapid progress on software engineering and ML research tasks, but these advances often assume access to a working runnable environment. For research artifacts released alongside published papers, setting up such an environment from a fresh machine remains a major bottleneck. Existing environment setup benchmarks do not cover the full scope of research-artifact deployment, which involves multi-language toolchains, system-level dependencies beyond containers (e.g., GPU/CUDA and kernel configurations), and legacy artifact compatibility. We introduce DeployBench, a multi-domain benchmark of 51 research-artifact deployment tasks spanning AI/ML, computer systems, and scientific computing, covering all these dimensions. Each task is verified by a hidden pipeline that executes the paper’s designated experiment and checks its outputs. Evaluating five state-of-the-art LLMs with OpenHands yields pass-rates from 7.8%-51.0%. Failures are dominated by a completion-judgment problem: 102 of 181 are agent-terminated self-stops, where the agent’s pre-finish checks validate a different or weaker target than the paper-specific task requires. DeployBench highlights the gap between current agents and autonomous deployment, and offers a realistic testbed for scientific research agents.
%U https://aclanthology.org/2026.findings-emnlp.707/
%P 14848-14866
Markdown (Informal)
[DeployBench : Benchmarking LLM Agents for Research Artifact Deployment](https://aclanthology.org/2026.findings-emnlp.707/) (Wang et al., Findings 2026)
ACL
- Yuanli Wang, Yaoyao Qian, Yue Zhang, Hanhan Zhou, Jindan Huang, Tianfu Fu, Qiuyang Mang, Huanzhi Mao, Wenhao Chai, Wendong Fan, and Liqiang Jing. 2026. DeployBench : Benchmarking LLM Agents for Research Artifact Deployment. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 14848–14866, Budapest, Hungary. Association for Computational Linguistics.