@inproceedings{fang-etal-2026-zserve,
title = "{ZSERVE}: An Adaptive Framework for Optimizing Large Language Models Serving in Heterogeneous Infrastructures",
author = "Fang, Taosong and
Ma, Zhengzhao and
Chen, Jiawei and
Yang, Yushen and
Zhou, Kaijie and
Zheng, Jia and
Han, Xianpei and
Sun, Le and
Lin, Hongyu and
Lu, Yaojie",
editor = "Agrawal, Sweta and
Abad, Alberto",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: System Demonstrations",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-demos.33/",
pages = "330--337",
ISBN = "979-8-89176-461-3",
abstract = "We introduce ZServe, an adaptive framework designed to optimize the deployment of large language models (LLMs) in production environments. ZServe tackles critical challenges in LLMs serving with three key innovations: (1) Resource-aware Parameter Mining, which automatically identifies optimal hardware configurations for various accelerators, eliminating the need for manual tuning while maximizing performance; (2) Task-aware Dynamic Reconfiguration, which seamlessly switches between SLO-centric and throughput-centric serving modes based on real-time workload characteristics without service disruption; and (3) A Unified Monitoring and Scaling Interface, which simplifies deployment management across heterogeneous infrastructures. In contrast to static frameworks like RayServe and Xinference, ZServe introduces dynamic runtime adaptation, leveraging novel queue handoff and memory-efficient state migration to switch serving modes without dropping in-flight requests. Experimental results with LLMs from 7B to 70B parameters show that ZServe achieves up to 45{\%} higher throughput during peak periods while preserving latency targets during moderate traffic, demonstrating scalability across single-node and multi-node deployments."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="fang-etal-2026-zserve">
<titleInfo>
<title>ZSERVE: An Adaptive Framework for Optimizing Large Language Models Serving in Heterogeneous Infrastructures</title>
</titleInfo>
<name type="personal">
<namePart type="given">Taosong</namePart>
<namePart type="family">Fang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhengzhao</namePart>
<namePart type="family">Ma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiawei</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yushen</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kaijie</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jia</namePart>
<namePart type="family">Zheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xianpei</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Le</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hongyu</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yaojie</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: System Demonstrations</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sweta</namePart>
<namePart type="family">Agrawal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alberto</namePart>
<namePart type="family">Abad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-461-3</identifier>
</relatedItem>
<abstract>We introduce ZServe, an adaptive framework designed to optimize the deployment of large language models (LLMs) in production environments. ZServe tackles critical challenges in LLMs serving with three key innovations: (1) Resource-aware Parameter Mining, which automatically identifies optimal hardware configurations for various accelerators, eliminating the need for manual tuning while maximizing performance; (2) Task-aware Dynamic Reconfiguration, which seamlessly switches between SLO-centric and throughput-centric serving modes based on real-time workload characteristics without service disruption; and (3) A Unified Monitoring and Scaling Interface, which simplifies deployment management across heterogeneous infrastructures. In contrast to static frameworks like RayServe and Xinference, ZServe introduces dynamic runtime adaptation, leveraging novel queue handoff and memory-efficient state migration to switch serving modes without dropping in-flight requests. Experimental results with LLMs from 7B to 70B parameters show that ZServe achieves up to 45% higher throughput during peak periods while preserving latency targets during moderate traffic, demonstrating scalability across single-node and multi-node deployments.</abstract>
<identifier type="citekey">fang-etal-2026-zserve</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-demos.33/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>330</start>
<end>337</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T ZSERVE: An Adaptive Framework for Optimizing Large Language Models Serving in Heterogeneous Infrastructures
%A Fang, Taosong
%A Ma, Zhengzhao
%A Chen, Jiawei
%A Yang, Yushen
%A Zhou, Kaijie
%A Zheng, Jia
%A Han, Xianpei
%A Sun, Le
%A Lin, Hongyu
%A Lu, Yaojie
%Y Agrawal, Sweta
%Y Abad, Alberto
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: System Demonstrations
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-461-3
%F fang-etal-2026-zserve
%X We introduce ZServe, an adaptive framework designed to optimize the deployment of large language models (LLMs) in production environments. ZServe tackles critical challenges in LLMs serving with three key innovations: (1) Resource-aware Parameter Mining, which automatically identifies optimal hardware configurations for various accelerators, eliminating the need for manual tuning while maximizing performance; (2) Task-aware Dynamic Reconfiguration, which seamlessly switches between SLO-centric and throughput-centric serving modes based on real-time workload characteristics without service disruption; and (3) A Unified Monitoring and Scaling Interface, which simplifies deployment management across heterogeneous infrastructures. In contrast to static frameworks like RayServe and Xinference, ZServe introduces dynamic runtime adaptation, leveraging novel queue handoff and memory-efficient state migration to switch serving modes without dropping in-flight requests. Experimental results with LLMs from 7B to 70B parameters show that ZServe achieves up to 45% higher throughput during peak periods while preserving latency targets during moderate traffic, demonstrating scalability across single-node and multi-node deployments.
%U https://aclanthology.org/2026.emnlp-demos.33/
%P 330-337
Markdown (Informal)
[ZSERVE: An Adaptive Framework for Optimizing Large Language Models Serving in Heterogeneous Infrastructures](https://aclanthology.org/2026.emnlp-demos.33/) (Fang et al., EMNLP 2026)
ACL
- Taosong Fang, Zhengzhao Ma, Jiawei Chen, Yushen Yang, Kaijie Zhou, Jia Zheng, Xianpei Han, Le Sun, Hongyu Lin, and Yaojie Lu. 2026. ZSERVE: An Adaptive Framework for Optimizing Large Language Models Serving in Heterogeneous Infrastructures. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: System Demonstrations, pages 330–337, Budapest, Hungary. Association for Computational Linguistics.