@inproceedings{zhang-etal-2026-vfpo,
title = "{VFPO}: Variance-aware Negative Feedback Policy Optimization for Mitigating Entropy Collapse via Group Statistics",
author = "Zhang, Bo and
Zeng, Zinan and
Zhou, Juntong and
Chong, Dading and
Ding, Zhenyu and
Wu, Dengpeng and
Ding, Ning and
Zhou, Peilin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2449/",
pages = "52561--52588",
ISBN = "979-8-89176-462-0",
abstract = "Reinforcement Learning with Verifiable Rewards (RLVR) has significantly advanced the reasoning capabilities of Large Language Models (LLMs), while being prone to entropy collapse, a rapid degradation of policy entropy that forces the model into premature convergence and limits its performance ceiling. However, existing solutions for mitigating entropy collapse often struggle to reconcile the trade-off between exploration capability and convergence efficiency. To address this, we propose Variance-aware Negative Feedback Policy Optimization (VFPO), a framework that dynamically regulates the policy entropy through the lens of group statistics. We first mathematically prove that policy entropy is intrinsically coupled with the variance and mean of group probabilities for sampled trajectories. Leveraging this insight, VFPO operates via a dual-phase strategy: an initial exploration phase employs a closed-loop negative feedback mechanism via both sample-level and token-level reward modulation to maintain group variance and prevent the mean trajectory probability from saturating; then followed by an annealing phase that relaxes these constraints to facilitate deterministic convergence. Extensive experiments on models of distinct scales (1.5B, 3B, 7B) demonstrate that VFPO yields average improvements of 3.34{\%}, 6.52{\%}, and 6.03{\%}, respectively, over the DAPO baseline, and achieves state-of-the-art performance across diverse mathematical benchmarks, effectively expanding the exploration boundary while achieving superior reasoning accuracy. To facilitate reproducibility, we release the code, data and trained models at the link https://github.com/carpe1/VFPO."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-vfpo">
<titleInfo>
<title>VFPO: Variance-aware Negative Feedback Policy Optimization for Mitigating Entropy Collapse via Group Statistics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Bo</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zinan</namePart>
<namePart type="family">Zeng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Juntong</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dading</namePart>
<namePart type="family">Chong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhenyu</namePart>
<namePart type="family">Ding</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dengpeng</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ning</namePart>
<namePart type="family">Ding</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peilin</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Reinforcement Learning with Verifiable Rewards (RLVR) has significantly advanced the reasoning capabilities of Large Language Models (LLMs), while being prone to entropy collapse, a rapid degradation of policy entropy that forces the model into premature convergence and limits its performance ceiling. However, existing solutions for mitigating entropy collapse often struggle to reconcile the trade-off between exploration capability and convergence efficiency. To address this, we propose Variance-aware Negative Feedback Policy Optimization (VFPO), a framework that dynamically regulates the policy entropy through the lens of group statistics. We first mathematically prove that policy entropy is intrinsically coupled with the variance and mean of group probabilities for sampled trajectories. Leveraging this insight, VFPO operates via a dual-phase strategy: an initial exploration phase employs a closed-loop negative feedback mechanism via both sample-level and token-level reward modulation to maintain group variance and prevent the mean trajectory probability from saturating; then followed by an annealing phase that relaxes these constraints to facilitate deterministic convergence. Extensive experiments on models of distinct scales (1.5B, 3B, 7B) demonstrate that VFPO yields average improvements of 3.34%, 6.52%, and 6.03%, respectively, over the DAPO baseline, and achieves state-of-the-art performance across diverse mathematical benchmarks, effectively expanding the exploration boundary while achieving superior reasoning accuracy. To facilitate reproducibility, we release the code, data and trained models at the link https://github.com/carpe1/VFPO.</abstract>
<identifier type="citekey">zhang-etal-2026-vfpo</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2449/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>52561</start>
<end>52588</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T VFPO: Variance-aware Negative Feedback Policy Optimization for Mitigating Entropy Collapse via Group Statistics
%A Zhang, Bo
%A Zeng, Zinan
%A Zhou, Juntong
%A Chong, Dading
%A Ding, Zhenyu
%A Wu, Dengpeng
%A Ding, Ning
%A Zhou, Peilin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F zhang-etal-2026-vfpo
%X Reinforcement Learning with Verifiable Rewards (RLVR) has significantly advanced the reasoning capabilities of Large Language Models (LLMs), while being prone to entropy collapse, a rapid degradation of policy entropy that forces the model into premature convergence and limits its performance ceiling. However, existing solutions for mitigating entropy collapse often struggle to reconcile the trade-off between exploration capability and convergence efficiency. To address this, we propose Variance-aware Negative Feedback Policy Optimization (VFPO), a framework that dynamically regulates the policy entropy through the lens of group statistics. We first mathematically prove that policy entropy is intrinsically coupled with the variance and mean of group probabilities for sampled trajectories. Leveraging this insight, VFPO operates via a dual-phase strategy: an initial exploration phase employs a closed-loop negative feedback mechanism via both sample-level and token-level reward modulation to maintain group variance and prevent the mean trajectory probability from saturating; then followed by an annealing phase that relaxes these constraints to facilitate deterministic convergence. Extensive experiments on models of distinct scales (1.5B, 3B, 7B) demonstrate that VFPO yields average improvements of 3.34%, 6.52%, and 6.03%, respectively, over the DAPO baseline, and achieves state-of-the-art performance across diverse mathematical benchmarks, effectively expanding the exploration boundary while achieving superior reasoning accuracy. To facilitate reproducibility, we release the code, data and trained models at the link https://github.com/carpe1/VFPO.
%U https://aclanthology.org/2026.findings-emnlp.2449/
%P 52561-52588
Markdown (Informal)
[VFPO: Variance-aware Negative Feedback Policy Optimization for Mitigating Entropy Collapse via Group Statistics](https://aclanthology.org/2026.findings-emnlp.2449/) (Zhang et al., Findings 2026)
ACL
- Bo Zhang, Zinan Zeng, Juntong Zhou, Dading Chong, Zhenyu Ding, Dengpeng Wu, Ning Ding, and Peilin Zhou. 2026. VFPO: Variance-aware Negative Feedback Policy Optimization for Mitigating Entropy Collapse via Group Statistics. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 52561–52588, Budapest, Hungary. Association for Computational Linguistics.