@inproceedings{wei-etal-2026-predicting,
title = "Predicting {LLM} Training and Inference Runtime: A Survey of Methods, Datasets, and Open Challenges",
author = "Wei, Kangda and
Bi, Ziqian and
Molakalapalli, Ashish and
Duan, Haonan and
Zou, Xiaowei and
Du, Vincent and
Carter, Alex and
Huang, Ruihong",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.486/",
pages = "10186--10203",
ISBN = "979-8-89176-462-0",
abstract = "Large language models have become the dominant workload in modern GPU clusters, driving unprecedented demand for compute during both training and inference. Yet GPU clusters typically operate at only 30{--}50{\%} utilization, largely because schedulers rely on user-provided runtime estimates that are notoriously inaccurate. Running thousands of LLM inference requests through shared datacenters further compounds this underutilization. Accurate runtime prediction is a key enabler of LLM efficiency: it supports shortest-job-first scheduling, backfilling, dynamic resource allocation, and cost-aware instance selection, while providing NLP practitioners with reliable completion estimates for training runs and serving latency guarantees. However, LLM workloads pose unique prediction challenges: attention complexity scales quadratically with sequence length, autoregressive decoding produces variable-length outputs, and 3D parallelism creates intricate performance interactions. This survey systematically reviews runtime prediction methods with emphasis on LLM training and inference workloads, presenting a feature taxonomy, analyzing design choices of existing prediction methods, surveying public datasets and highlighting their limitations. We identify applications critical to LLM efficiency and key open challenges including generalization to rapidly evolving model architectures and hardwares."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wei-etal-2026-predicting">
<titleInfo>
<title>Predicting LLM Training and Inference Runtime: A Survey of Methods, Datasets, and Open Challenges</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kangda</namePart>
<namePart type="family">Wei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziqian</namePart>
<namePart type="family">Bi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ashish</namePart>
<namePart type="family">Molakalapalli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haonan</namePart>
<namePart type="family">Duan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaowei</namePart>
<namePart type="family">Zou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vincent</namePart>
<namePart type="family">Du</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Carter</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruihong</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large language models have become the dominant workload in modern GPU clusters, driving unprecedented demand for compute during both training and inference. Yet GPU clusters typically operate at only 30–50% utilization, largely because schedulers rely on user-provided runtime estimates that are notoriously inaccurate. Running thousands of LLM inference requests through shared datacenters further compounds this underutilization. Accurate runtime prediction is a key enabler of LLM efficiency: it supports shortest-job-first scheduling, backfilling, dynamic resource allocation, and cost-aware instance selection, while providing NLP practitioners with reliable completion estimates for training runs and serving latency guarantees. However, LLM workloads pose unique prediction challenges: attention complexity scales quadratically with sequence length, autoregressive decoding produces variable-length outputs, and 3D parallelism creates intricate performance interactions. This survey systematically reviews runtime prediction methods with emphasis on LLM training and inference workloads, presenting a feature taxonomy, analyzing design choices of existing prediction methods, surveying public datasets and highlighting their limitations. We identify applications critical to LLM efficiency and key open challenges including generalization to rapidly evolving model architectures and hardwares.</abstract>
<identifier type="citekey">wei-etal-2026-predicting</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.486/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>10186</start>
<end>10203</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Predicting LLM Training and Inference Runtime: A Survey of Methods, Datasets, and Open Challenges
%A Wei, Kangda
%A Bi, Ziqian
%A Molakalapalli, Ashish
%A Duan, Haonan
%A Zou, Xiaowei
%A Du, Vincent
%A Carter, Alex
%A Huang, Ruihong
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F wei-etal-2026-predicting
%X Large language models have become the dominant workload in modern GPU clusters, driving unprecedented demand for compute during both training and inference. Yet GPU clusters typically operate at only 30–50% utilization, largely because schedulers rely on user-provided runtime estimates that are notoriously inaccurate. Running thousands of LLM inference requests through shared datacenters further compounds this underutilization. Accurate runtime prediction is a key enabler of LLM efficiency: it supports shortest-job-first scheduling, backfilling, dynamic resource allocation, and cost-aware instance selection, while providing NLP practitioners with reliable completion estimates for training runs and serving latency guarantees. However, LLM workloads pose unique prediction challenges: attention complexity scales quadratically with sequence length, autoregressive decoding produces variable-length outputs, and 3D parallelism creates intricate performance interactions. This survey systematically reviews runtime prediction methods with emphasis on LLM training and inference workloads, presenting a feature taxonomy, analyzing design choices of existing prediction methods, surveying public datasets and highlighting their limitations. We identify applications critical to LLM efficiency and key open challenges including generalization to rapidly evolving model architectures and hardwares.
%U https://aclanthology.org/2026.findings-emnlp.486/
%P 10186-10203
Markdown (Informal)
[Predicting LLM Training and Inference Runtime: A Survey of Methods, Datasets, and Open Challenges](https://aclanthology.org/2026.findings-emnlp.486/) (Wei et al., Findings 2026)
ACL
- Kangda Wei, Ziqian Bi, Ashish Molakalapalli, Haonan Duan, Xiaowei Zou, Vincent Du, Alex Carter, and Ruihong Huang. 2026. Predicting LLM Training and Inference Runtime: A Survey of Methods, Datasets, and Open Challenges. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 10186–10203, Budapest, Hungary. Association for Computational Linguistics.