@inproceedings{wang-etal-2026-beyond-top,
title = "Beyond Top-k : Mass-Aware Support Sampling for On-Policy Distillation",
author = "Wang, Yu and
Chen, Xinran and
Li, Yuchen and
Honghai, Zhang and
Kong, Rui and
Chen, Jiamin and
Yan, Lingyong and
Kong, Linghe and
Ma, Chen and
Xiong, Haoyi and
Yin, Dawei",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.198/",
pages = "4540--4554",
ISBN = "979-8-89176-459-0",
abstract = "On-policy distillation reduces train-inference mismatch by supervising student-generated trajectories, but full-vocabulary token-level distribution matching remains expensive for large language models. A common approximation truncates reverse-KL distillation to the student{'}s top-$k$ tokens, which preserves efficiency but drops tail-token supervision and ignores support-level mass mismatch between the student and teacher. We analyze these approximation gaps and propose Stochastic Support Sampling with Teacher Mass Reweighting, which samples fixed-size supports from the student distribution and uses teacher support mass as a reliability weight for the conditional distillation loss. Under the same support budget, our method outperforms the strongest baseline by +1.4 hard-math average points on Qwen3-8B{\textrightarrow}Qwen3-1.7B-Base, and by +1.1 math average points and +2.8 code average points on DeepSeek-R1-Distill-Llama-8B{\textrightarrow}Meta-Llama-3.1-8B-Base."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-beyond-top">
<titleInfo>
<title>Beyond Top-k : Mass-Aware Support Sampling for On-Policy Distillation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinran</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuchen</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhang</namePart>
<namePart type="family">Honghai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rui</namePart>
<namePart type="family">Kong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiamin</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lingyong</namePart>
<namePart type="family">Yan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Linghe</namePart>
<namePart type="family">Kong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chen</namePart>
<namePart type="family">Ma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haoyi</namePart>
<namePart type="family">Xiong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dawei</namePart>
<namePart type="family">Yin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>On-policy distillation reduces train-inference mismatch by supervising student-generated trajectories, but full-vocabulary token-level distribution matching remains expensive for large language models. A common approximation truncates reverse-KL distillation to the student’s top-k tokens, which preserves efficiency but drops tail-token supervision and ignores support-level mass mismatch between the student and teacher. We analyze these approximation gaps and propose Stochastic Support Sampling with Teacher Mass Reweighting, which samples fixed-size supports from the student distribution and uses teacher support mass as a reliability weight for the conditional distillation loss. Under the same support budget, our method outperforms the strongest baseline by +1.4 hard-math average points on Qwen3-8B→Qwen3-1.7B-Base, and by +1.1 math average points and +2.8 code average points on DeepSeek-R1-Distill-Llama-8B→Meta-Llama-3.1-8B-Base.</abstract>
<identifier type="citekey">wang-etal-2026-beyond-top</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.198/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>4540</start>
<end>4554</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Top-k : Mass-Aware Support Sampling for On-Policy Distillation
%A Wang, Yu
%A Chen, Xinran
%A Li, Yuchen
%A Honghai, Zhang
%A Kong, Rui
%A Chen, Jiamin
%A Yan, Lingyong
%A Kong, Linghe
%A Ma, Chen
%A Xiong, Haoyi
%A Yin, Dawei
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F wang-etal-2026-beyond-top
%X On-policy distillation reduces train-inference mismatch by supervising student-generated trajectories, but full-vocabulary token-level distribution matching remains expensive for large language models. A common approximation truncates reverse-KL distillation to the student’s top-k tokens, which preserves efficiency but drops tail-token supervision and ignores support-level mass mismatch between the student and teacher. We analyze these approximation gaps and propose Stochastic Support Sampling with Teacher Mass Reweighting, which samples fixed-size supports from the student distribution and uses teacher support mass as a reliability weight for the conditional distillation loss. Under the same support budget, our method outperforms the strongest baseline by +1.4 hard-math average points on Qwen3-8B→Qwen3-1.7B-Base, and by +1.1 math average points and +2.8 code average points on DeepSeek-R1-Distill-Llama-8B→Meta-Llama-3.1-8B-Base.
%U https://aclanthology.org/2026.emnlp-main.198/
%P 4540-4554
Markdown (Informal)
[Beyond Top-k : Mass-Aware Support Sampling for On-Policy Distillation](https://aclanthology.org/2026.emnlp-main.198/) (Wang et al., EMNLP 2026)
ACL
- Yu Wang, Xinran Chen, Yuchen Li, Zhang Honghai, Rui Kong, Jiamin Chen, Lingyong Yan, Linghe Kong, Chen Ma, Haoyi Xiong, and Dawei Yin. 2026. Beyond Top-k : Mass-Aware Support Sampling for On-Policy Distillation. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 4540–4554, Budapest, Hungary. Association for Computational Linguistics.