@inproceedings{liu-etal-2026-teacher,
title = "Teacher-Guided Policy Optimization for On-Policy Reasoning Distillation under Large Policy Divergence",
author = "Liu, Xinyu and
Jiao, Kechen and
Xiao, Chunyang and
Zhao, Runsong and
Ruan, Junhao and
Li, Bei and
Liu, Jiahao and
Wang, Qifan and
Chen, Xin and
Wang, Jingang and
Wang, Chenglong and
Xiao, Tong and
Zhu, JingBo",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1172/",
pages = "24801--24816",
ISBN = "979-8-89176-462-0",
abstract = "On-policy distillation (OPD) has emerged as a promising paradigm for reasoning-oriented post-training of large language models (LLMs), especially when combined with reinforcement learning from verifiable rewards (RLVR). Existing OPD methods rely on reverse KL (RKL)-based teacher supervision over trajectories sampled from the student policy. However, we identify a critical limitation: under large teacher{--}student policy divergence, RL-driven exploration often produces trajectories that the teacher assigns low probability, resulting in uninformative negative feedback. To address this, we propose Teacher-Guided Policy Optimization (TGPO). Unlike existing OPD methods that evaluate student-sampled actions using the teacher, TGPO directly guides next-token generation using the teacher{'}s predicted distribution on student-generated contexts, yielding an on-policy forward-KL objective. Combined with RLVR, this guidance steers exploration toward improved continuations. Experiments on reasoning benchmarks show that TGPO outperforms existing RKL-based OPD methods and remains robust across different teacher models. Furthermore, we find that teacher top-1 predictions suffice for effective guidance, without requiring the full teacher distribution."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-teacher">
<titleInfo>
<title>Teacher-Guided Policy Optimization for On-Policy Reasoning Distillation under Large Policy Divergence</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kechen</namePart>
<namePart type="family">Jiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chunyang</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Runsong</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junhao</namePart>
<namePart type="family">Ruan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bei</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiahao</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qifan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingang</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chenglong</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">JingBo</namePart>
<namePart type="family">Zhu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>On-policy distillation (OPD) has emerged as a promising paradigm for reasoning-oriented post-training of large language models (LLMs), especially when combined with reinforcement learning from verifiable rewards (RLVR). Existing OPD methods rely on reverse KL (RKL)-based teacher supervision over trajectories sampled from the student policy. However, we identify a critical limitation: under large teacher–student policy divergence, RL-driven exploration often produces trajectories that the teacher assigns low probability, resulting in uninformative negative feedback. To address this, we propose Teacher-Guided Policy Optimization (TGPO). Unlike existing OPD methods that evaluate student-sampled actions using the teacher, TGPO directly guides next-token generation using the teacher’s predicted distribution on student-generated contexts, yielding an on-policy forward-KL objective. Combined with RLVR, this guidance steers exploration toward improved continuations. Experiments on reasoning benchmarks show that TGPO outperforms existing RKL-based OPD methods and remains robust across different teacher models. Furthermore, we find that teacher top-1 predictions suffice for effective guidance, without requiring the full teacher distribution.</abstract>
<identifier type="citekey">liu-etal-2026-teacher</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1172/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>24801</start>
<end>24816</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Teacher-Guided Policy Optimization for On-Policy Reasoning Distillation under Large Policy Divergence
%A Liu, Xinyu
%A Jiao, Kechen
%A Xiao, Chunyang
%A Zhao, Runsong
%A Ruan, Junhao
%A Li, Bei
%A Liu, Jiahao
%A Wang, Qifan
%A Chen, Xin
%A Wang, Jingang
%A Wang, Chenglong
%A Xiao, Tong
%A Zhu, JingBo
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F liu-etal-2026-teacher
%X On-policy distillation (OPD) has emerged as a promising paradigm for reasoning-oriented post-training of large language models (LLMs), especially when combined with reinforcement learning from verifiable rewards (RLVR). Existing OPD methods rely on reverse KL (RKL)-based teacher supervision over trajectories sampled from the student policy. However, we identify a critical limitation: under large teacher–student policy divergence, RL-driven exploration often produces trajectories that the teacher assigns low probability, resulting in uninformative negative feedback. To address this, we propose Teacher-Guided Policy Optimization (TGPO). Unlike existing OPD methods that evaluate student-sampled actions using the teacher, TGPO directly guides next-token generation using the teacher’s predicted distribution on student-generated contexts, yielding an on-policy forward-KL objective. Combined with RLVR, this guidance steers exploration toward improved continuations. Experiments on reasoning benchmarks show that TGPO outperforms existing RKL-based OPD methods and remains robust across different teacher models. Furthermore, we find that teacher top-1 predictions suffice for effective guidance, without requiring the full teacher distribution.
%U https://aclanthology.org/2026.findings-emnlp.1172/
%P 24801-24816
Markdown (Informal)
[Teacher-Guided Policy Optimization for On-Policy Reasoning Distillation under Large Policy Divergence](https://aclanthology.org/2026.findings-emnlp.1172/) (Liu et al., Findings 2026)
ACL
- Xinyu Liu, Kechen Jiao, Chunyang Xiao, Runsong Zhao, Junhao Ruan, Bei Li, Jiahao Liu, Qifan Wang, Xin Chen, Jingang Wang, Chenglong Wang, Tong Xiao, and JingBo Zhu. 2026. Teacher-Guided Policy Optimization for On-Policy Reasoning Distillation under Large Policy Divergence. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 24801–24816, Budapest, Hungary. Association for Computational Linguistics.