@inproceedings{zhang-etal-2026-kdflow,
title = "{KDF}low: A User-Friendly and Efficient Knowledge Distillation Framework for Large Language Models",
author = "Zhang, Songming and
Zhang, Xue and
Zhang, Tong and
Hu, Bojie and
Chen, Yufeng and
Xu, Jinan",
editor = "Agrawal, Sweta and
Abad, Alberto",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: System Demonstrations",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-demos.31/",
pages = "310--319",
ISBN = "979-8-89176-461-3",
abstract = "Knowledge distillation (KD) is widely used to compress and post-train large language models (LLMs), yet many existing frameworks execute teacher inference with the same training-oriented backend as student optimization, leading to suboptimal efficiency. In this paper, we propose \textbf{KDFlow}, a novel framework for LLM distillation that features a \textbf{decoupled architecture} and employs SGLang for teacher inference. KDFlow combines SGLang for teacher inference with PyTorch FSDP2 for student optimization, allowing each model to run on a backend tailored to its workload. To enable \textbf{efficient full-vocabulary distillation} in this decoupled architecture, KDFlow transfers the teacher{'}s final hidden states via Ray{'}s object store and recomputes teacher logits on each student worker using a frozen copy of the teacher{'}s output head. Furthermore, our framework supports both off-policy and on-policy distillation and incorporates cross-tokenizer algorithms through highly extensible and user-friendly APIs. Experiments show that KDFlow achieves a \textbf{1.44$\times$ to 6.36$\times$} speedup over MS-SWIFT in off-policy distillation and a \textbf{1.43$\times$ to 1.75$\times$} speedup over verl in on-policy distillation. KDFlow further scales to 64 GPUs, achieving 3.68$\times$ and 2.52$\times$ strong-scaling speedups in two representative model configurations. The code and documentation are publicly available. The code and documentation are publicly available."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-kdflow">
<titleInfo>
<title>KDFlow: A User-Friendly and Efficient Knowledge Distillation Framework for Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Songming</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bojie</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yufeng</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jinan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: System Demonstrations</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sweta</namePart>
<namePart type="family">Agrawal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alberto</namePart>
<namePart type="family">Abad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-461-3</identifier>
</relatedItem>
<abstract>Knowledge distillation (KD) is widely used to compress and post-train large language models (LLMs), yet many existing frameworks execute teacher inference with the same training-oriented backend as student optimization, leading to suboptimal efficiency. In this paper, we propose KDFlow, a novel framework for LLM distillation that features a decoupled architecture and employs SGLang for teacher inference. KDFlow combines SGLang for teacher inference with PyTorch FSDP2 for student optimization, allowing each model to run on a backend tailored to its workload. To enable efficient full-vocabulary distillation in this decoupled architecture, KDFlow transfers the teacher’s final hidden states via Ray’s object store and recomputes teacher logits on each student worker using a frozen copy of the teacher’s output head. Furthermore, our framework supports both off-policy and on-policy distillation and incorporates cross-tokenizer algorithms through highly extensible and user-friendly APIs. Experiments show that KDFlow achieves a 1.44\times to 6.36\times speedup over MS-SWIFT in off-policy distillation and a 1.43\times to 1.75\times speedup over verl in on-policy distillation. KDFlow further scales to 64 GPUs, achieving 3.68\times and 2.52\times strong-scaling speedups in two representative model configurations. The code and documentation are publicly available. The code and documentation are publicly available.</abstract>
<identifier type="citekey">zhang-etal-2026-kdflow</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-demos.31/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>310</start>
<end>319</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T KDFlow: A User-Friendly and Efficient Knowledge Distillation Framework for Large Language Models
%A Zhang, Songming
%A Zhang, Xue
%A Zhang, Tong
%A Hu, Bojie
%A Chen, Yufeng
%A Xu, Jinan
%Y Agrawal, Sweta
%Y Abad, Alberto
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: System Demonstrations
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-461-3
%F zhang-etal-2026-kdflow
%X Knowledge distillation (KD) is widely used to compress and post-train large language models (LLMs), yet many existing frameworks execute teacher inference with the same training-oriented backend as student optimization, leading to suboptimal efficiency. In this paper, we propose KDFlow, a novel framework for LLM distillation that features a decoupled architecture and employs SGLang for teacher inference. KDFlow combines SGLang for teacher inference with PyTorch FSDP2 for student optimization, allowing each model to run on a backend tailored to its workload. To enable efficient full-vocabulary distillation in this decoupled architecture, KDFlow transfers the teacher’s final hidden states via Ray’s object store and recomputes teacher logits on each student worker using a frozen copy of the teacher’s output head. Furthermore, our framework supports both off-policy and on-policy distillation and incorporates cross-tokenizer algorithms through highly extensible and user-friendly APIs. Experiments show that KDFlow achieves a 1.44\times to 6.36\times speedup over MS-SWIFT in off-policy distillation and a 1.43\times to 1.75\times speedup over verl in on-policy distillation. KDFlow further scales to 64 GPUs, achieving 3.68\times and 2.52\times strong-scaling speedups in two representative model configurations. The code and documentation are publicly available. The code and documentation are publicly available.
%U https://aclanthology.org/2026.emnlp-demos.31/
%P 310-319
Markdown (Informal)
[KDFlow: A User-Friendly and Efficient Knowledge Distillation Framework for Large Language Models](https://aclanthology.org/2026.emnlp-demos.31/) (Zhang et al., EMNLP 2026)
ACL