@inproceedings{zhao-etal-2026-poweropd,
title = "{P}ower{OPD}: Stabilizing On-Policy Distillation with Bounded Power Transformation",
author = "Zhao, Anhao and
Tong, Junlong and
Fan, Yingqi and
Nie, Ping and
Li, Wenjie and
Shen, Xiaoyu",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2129/",
pages = "48414--48427",
ISBN = "979-8-89176-459-0",
abstract = "Standard on-policy distillation (OPD) for large language models estimates the reverse-KL objective using student-sampled tokens, yielding an unbiased single-sample Monte Carlo estimator that avoids vocabulary-wide computation. However, we show that this estimator suffers from severe training pathologies in practice: sample inefficiency, unstable generation dynamics, and a substantial performance gap compared to exact full-vocabulary OPD. Reward-level diagnosis traces these pathologies to the log-ratio reward, which is unbounded by construction, producing extremely high-variance gradients concentrated at early positions and persisting throughout training; standard post-hoc scaling fail as they operate only after this distortion occurs. To solve this problem, we propose *PowerOPD*: a family of natively bounded, sign-consistent rewards from the Box-Cox power transformation, parameterized by $\alpha > 0$, of which the log-ratio is the degenerate $\alpha \to 0$ limit. Across six mathematical reasoning benchmarks and four Qwen3 teacher{--}student pairs, PowerOPD achieves benchmark-averaged Avg@8/Pass@8 gains of up to $\mathbf{+6.37/+5.71}$ over vanilla OPD, $\mathbf{+3.01/+3.54}$ over post-hoc stabilization, and $\mathbf{+2.59/+8.90}$ over full-vocabulary OPD, while reducing wall-clock time by 59.2{\%} and peak GPU memory by 23.1{\%}. Larger $\alpha$ generally improves accuracy, consistently shortens responses, and keeps gradient norms more than $\mathbf{3{,}000\times}$ smaller than vanilla OPD."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhao-etal-2026-poweropd">
<titleInfo>
<title>PowerOPD: Stabilizing On-Policy Distillation with Bounded Power Transformation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Anhao</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junlong</namePart>
<namePart type="family">Tong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yingqi</namePart>
<namePart type="family">Fan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ping</namePart>
<namePart type="family">Nie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenjie</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaoyu</namePart>
<namePart type="family">Shen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Standard on-policy distillation (OPD) for large language models estimates the reverse-KL objective using student-sampled tokens, yielding an unbiased single-sample Monte Carlo estimator that avoids vocabulary-wide computation. However, we show that this estimator suffers from severe training pathologies in practice: sample inefficiency, unstable generation dynamics, and a substantial performance gap compared to exact full-vocabulary OPD. Reward-level diagnosis traces these pathologies to the log-ratio reward, which is unbounded by construction, producing extremely high-variance gradients concentrated at early positions and persisting throughout training; standard post-hoc scaling fail as they operate only after this distortion occurs. To solve this problem, we propose *PowerOPD*: a family of natively bounded, sign-consistent rewards from the Box-Cox power transformation, parameterized by α > 0, of which the log-ratio is the degenerate α 0 limit. Across six mathematical reasoning benchmarks and four Qwen3 teacher–student pairs, PowerOPD achieves benchmark-averaged Avg@8/Pass@8 gains of up to \mathbf+6.37/+5.71 over vanilla OPD, \mathbf+3.01/+3.54 over post-hoc stabilization, and \mathbf+2.59/+8.90 over full-vocabulary OPD, while reducing wall-clock time by 59.2% and peak GPU memory by 23.1%. Larger α generally improves accuracy, consistently shortens responses, and keeps gradient norms more than \mathbf3,000\times smaller than vanilla OPD.</abstract>
<identifier type="citekey">zhao-etal-2026-poweropd</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2129/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>48414</start>
<end>48427</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T PowerOPD: Stabilizing On-Policy Distillation with Bounded Power Transformation
%A Zhao, Anhao
%A Tong, Junlong
%A Fan, Yingqi
%A Nie, Ping
%A Li, Wenjie
%A Shen, Xiaoyu
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F zhao-etal-2026-poweropd
%X Standard on-policy distillation (OPD) for large language models estimates the reverse-KL objective using student-sampled tokens, yielding an unbiased single-sample Monte Carlo estimator that avoids vocabulary-wide computation. However, we show that this estimator suffers from severe training pathologies in practice: sample inefficiency, unstable generation dynamics, and a substantial performance gap compared to exact full-vocabulary OPD. Reward-level diagnosis traces these pathologies to the log-ratio reward, which is unbounded by construction, producing extremely high-variance gradients concentrated at early positions and persisting throughout training; standard post-hoc scaling fail as they operate only after this distortion occurs. To solve this problem, we propose *PowerOPD*: a family of natively bounded, sign-consistent rewards from the Box-Cox power transformation, parameterized by α > 0, of which the log-ratio is the degenerate α 0 limit. Across six mathematical reasoning benchmarks and four Qwen3 teacher–student pairs, PowerOPD achieves benchmark-averaged Avg@8/Pass@8 gains of up to \mathbf+6.37/+5.71 over vanilla OPD, \mathbf+3.01/+3.54 over post-hoc stabilization, and \mathbf+2.59/+8.90 over full-vocabulary OPD, while reducing wall-clock time by 59.2% and peak GPU memory by 23.1%. Larger α generally improves accuracy, consistently shortens responses, and keeps gradient norms more than \mathbf3,000\times smaller than vanilla OPD.
%U https://aclanthology.org/2026.emnlp-main.2129/
%P 48414-48427
Markdown (Informal)
[PowerOPD: Stabilizing On-Policy Distillation with Bounded Power Transformation](https://aclanthology.org/2026.emnlp-main.2129/) (Zhao et al., EMNLP 2026)
ACL