@inproceedings{xu-etal-2026-beyond-grpo,
title = "Beyond {GRPO} and On-Policy Distillation: An Empirical Sparse-to-Dense Reward Principle for Language-Model Post-Training",
author = "Xu, Yuanda and
Sang, Hejian and
Zhou, Zhengze and
He, Ran and
Wang, Zhipeng and
Geramifard, Alborz",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.639/",
pages = "13392--13408",
ISBN = "979-8-89176-462-0",
abstract = "In settings where labeled verifiable training data is the binding constraint, each checked example should be allocated to the model and reward density where it is most informative. We identify a reward-density principle: sparse sequence-level reward is most useful on models that can explore and discover better behavior, while dense token-level teacher supervision is better suited for compressing that behavior into a smaller deployment model; accordingly, scarce labeled data should be used upstream on the strongest available teacher, then transferred downstream as dense supervision.\par We evaluate this rule through a four-stage workflow: teacher RL, forward-KL warmup, on-policy distillation, optional post-bridge student RL on verifiable math and code generation with Qwen3, Llama, and Qwen2.5 models. At fixed Qwen3-1.7B deployment-student size, an RL-improved 8B teacher distilled through the dense bridge outperforms direct GRPO on the same student (79.3{\%} vs. 75.9{\%} on MATH; 25.2{\%} vs. 19.8{\%} on AIME2024, avg@16), while transfer from the same teacher before RL underperforms. A component ablation confirms that each stage is load-bearing: the RL-improved teacher adds 7.8 absolute points on MATH over a raw teacher of the same size, the forward-KL warmup adds 1.7, and on-policy distillation adds 3.3. The teacher-quality ordering: raw-teacher transfer $<$ direct GRPO $<$ RL-teacher transfer replicates on Llama-3.1-8B-Instruct with a Llama-3.3-70B-Instruct teacher and on Qwen2.5-7B-Instruct for code generation with a Qwen2.5-32B-Instruct teacher, where the dense bridge improves HumanEval and MBPP by 3.3 and 3.8 absolute points, respectively, over direct student GRPO. The operational lesson is to avoid spending scarce labeled data on the least prepared policy: use sparse reward for teacher-side discovery, dense transfer for student compression, and student-side sparse reward only after the bridge. The OPD implementation used in our experiments is available at https://github.com/HJSang/OPSD{\_}OnPolicyDistillation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="xu-etal-2026-beyond-grpo">
<titleInfo>
<title>Beyond GRPO and On-Policy Distillation: An Empirical Sparse-to-Dense Reward Principle for Language-Model Post-Training</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yuanda</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hejian</namePart>
<namePart type="family">Sang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhengze</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ran</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhipeng</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alborz</namePart>
<namePart type="family">Geramifard</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>In settings where labeled verifiable training data is the binding constraint, each checked example should be allocated to the model and reward density where it is most informative. We identify a reward-density principle: sparse sequence-level reward is most useful on models that can explore and discover better behavior, while dense token-level teacher supervision is better suited for compressing that behavior into a smaller deployment model; accordingly, scarce labeled data should be used upstream on the strongest available teacher, then transferred downstream as dense supervision.\par We evaluate this rule through a four-stage workflow: teacher RL, forward-KL warmup, on-policy distillation, optional post-bridge student RL on verifiable math and code generation with Qwen3, Llama, and Qwen2.5 models. At fixed Qwen3-1.7B deployment-student size, an RL-improved 8B teacher distilled through the dense bridge outperforms direct GRPO on the same student (79.3% vs. 75.9% on MATH; 25.2% vs. 19.8% on AIME2024, avg@16), while transfer from the same teacher before RL underperforms. A component ablation confirms that each stage is load-bearing: the RL-improved teacher adds 7.8 absolute points on MATH over a raw teacher of the same size, the forward-KL warmup adds 1.7, and on-policy distillation adds 3.3. The teacher-quality ordering: raw-teacher transfer < direct GRPO < RL-teacher transfer replicates on Llama-3.1-8B-Instruct with a Llama-3.3-70B-Instruct teacher and on Qwen2.5-7B-Instruct for code generation with a Qwen2.5-32B-Instruct teacher, where the dense bridge improves HumanEval and MBPP by 3.3 and 3.8 absolute points, respectively, over direct student GRPO. The operational lesson is to avoid spending scarce labeled data on the least prepared policy: use sparse reward for teacher-side discovery, dense transfer for student compression, and student-side sparse reward only after the bridge. The OPD implementation used in our experiments is available at https://github.com/HJSang/OPSD_OnPolicyDistillation.</abstract>
<identifier type="citekey">xu-etal-2026-beyond-grpo</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.639/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>13392</start>
<end>13408</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond GRPO and On-Policy Distillation: An Empirical Sparse-to-Dense Reward Principle for Language-Model Post-Training
%A Xu, Yuanda
%A Sang, Hejian
%A Zhou, Zhengze
%A He, Ran
%A Wang, Zhipeng
%A Geramifard, Alborz
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F xu-etal-2026-beyond-grpo
%X In settings where labeled verifiable training data is the binding constraint, each checked example should be allocated to the model and reward density where it is most informative. We identify a reward-density principle: sparse sequence-level reward is most useful on models that can explore and discover better behavior, while dense token-level teacher supervision is better suited for compressing that behavior into a smaller deployment model; accordingly, scarce labeled data should be used upstream on the strongest available teacher, then transferred downstream as dense supervision.\par We evaluate this rule through a four-stage workflow: teacher RL, forward-KL warmup, on-policy distillation, optional post-bridge student RL on verifiable math and code generation with Qwen3, Llama, and Qwen2.5 models. At fixed Qwen3-1.7B deployment-student size, an RL-improved 8B teacher distilled through the dense bridge outperforms direct GRPO on the same student (79.3% vs. 75.9% on MATH; 25.2% vs. 19.8% on AIME2024, avg@16), while transfer from the same teacher before RL underperforms. A component ablation confirms that each stage is load-bearing: the RL-improved teacher adds 7.8 absolute points on MATH over a raw teacher of the same size, the forward-KL warmup adds 1.7, and on-policy distillation adds 3.3. The teacher-quality ordering: raw-teacher transfer < direct GRPO < RL-teacher transfer replicates on Llama-3.1-8B-Instruct with a Llama-3.3-70B-Instruct teacher and on Qwen2.5-7B-Instruct for code generation with a Qwen2.5-32B-Instruct teacher, where the dense bridge improves HumanEval and MBPP by 3.3 and 3.8 absolute points, respectively, over direct student GRPO. The operational lesson is to avoid spending scarce labeled data on the least prepared policy: use sparse reward for teacher-side discovery, dense transfer for student compression, and student-side sparse reward only after the bridge. The OPD implementation used in our experiments is available at https://github.com/HJSang/OPSD_OnPolicyDistillation.
%U https://aclanthology.org/2026.findings-emnlp.639/
%P 13392-13408
Markdown (Informal)
[Beyond GRPO and On-Policy Distillation: An Empirical Sparse-to-Dense Reward Principle for Language-Model Post-Training](https://aclanthology.org/2026.findings-emnlp.639/) (Xu et al., Findings 2026)
ACL