@inproceedings{gong-etal-2026-stop,
title = "Stop Rolling the Dice: Active Group Relative Policy Optimization via Intrinsic Metacognition",
author = "Gong, Zheng and
Sun, Ying and
Peng, Hao and
Yu, Philip S.",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.614/",
pages = "12915--12927",
ISBN = "979-8-89176-462-0",
abstract = "Group Relative Policy Optimization (GRPO) is an attractive reinforcement learning paradigm for Large Reasoning Models because its groupwise relative advantages eliminate the need for a value network, but its efficiency is severely limited by zero-variance queries whose rollouts are uniformly correct or uniformly incorrect and therefore consume expensive Chain-of-Thought generation without providing useful learning signals. Existing remedies typically rely on offline difficulty labels or external capability trackers, which either fail to follow the model{'}s rapidly evolving competence or compromise the end-to-end simplicity of GRPO. We propose Active-GRPO, a prompt-free extension that equips the policy with intrinsic fuzzy metacognition via a lightweight confidence head attached to the final prefill representation. The head predicts ordinal confidence bins and is jointly calibrated against empirical rollout outcomes through a dual-objective optimization framework. It further enables a multi-criterion voting mechanism that filters uninformative queries before rollout generation, thereby concentrating compute on samples near the learning boundary with negligible overhead and without perturbing the reasoning distribution. Experiments on mathematical and logical reasoning benchmarks show that Active-GRPO reduces training computation while improving both reasoning accuracy and intrinsic calibration quality."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="gong-etal-2026-stop">
<titleInfo>
<title>Stop Rolling the Dice: Active Group Relative Policy Optimization via Intrinsic Metacognition</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zheng</namePart>
<namePart type="family">Gong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ying</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hao</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Philip</namePart>
<namePart type="given">S</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Group Relative Policy Optimization (GRPO) is an attractive reinforcement learning paradigm for Large Reasoning Models because its groupwise relative advantages eliminate the need for a value network, but its efficiency is severely limited by zero-variance queries whose rollouts are uniformly correct or uniformly incorrect and therefore consume expensive Chain-of-Thought generation without providing useful learning signals. Existing remedies typically rely on offline difficulty labels or external capability trackers, which either fail to follow the model’s rapidly evolving competence or compromise the end-to-end simplicity of GRPO. We propose Active-GRPO, a prompt-free extension that equips the policy with intrinsic fuzzy metacognition via a lightweight confidence head attached to the final prefill representation. The head predicts ordinal confidence bins and is jointly calibrated against empirical rollout outcomes through a dual-objective optimization framework. It further enables a multi-criterion voting mechanism that filters uninformative queries before rollout generation, thereby concentrating compute on samples near the learning boundary with negligible overhead and without perturbing the reasoning distribution. Experiments on mathematical and logical reasoning benchmarks show that Active-GRPO reduces training computation while improving both reasoning accuracy and intrinsic calibration quality.</abstract>
<identifier type="citekey">gong-etal-2026-stop</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.614/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>12915</start>
<end>12927</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Stop Rolling the Dice: Active Group Relative Policy Optimization via Intrinsic Metacognition
%A Gong, Zheng
%A Sun, Ying
%A Peng, Hao
%A Yu, Philip S.
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F gong-etal-2026-stop
%X Group Relative Policy Optimization (GRPO) is an attractive reinforcement learning paradigm for Large Reasoning Models because its groupwise relative advantages eliminate the need for a value network, but its efficiency is severely limited by zero-variance queries whose rollouts are uniformly correct or uniformly incorrect and therefore consume expensive Chain-of-Thought generation without providing useful learning signals. Existing remedies typically rely on offline difficulty labels or external capability trackers, which either fail to follow the model’s rapidly evolving competence or compromise the end-to-end simplicity of GRPO. We propose Active-GRPO, a prompt-free extension that equips the policy with intrinsic fuzzy metacognition via a lightweight confidence head attached to the final prefill representation. The head predicts ordinal confidence bins and is jointly calibrated against empirical rollout outcomes through a dual-objective optimization framework. It further enables a multi-criterion voting mechanism that filters uninformative queries before rollout generation, thereby concentrating compute on samples near the learning boundary with negligible overhead and without perturbing the reasoning distribution. Experiments on mathematical and logical reasoning benchmarks show that Active-GRPO reduces training computation while improving both reasoning accuracy and intrinsic calibration quality.
%U https://aclanthology.org/2026.findings-emnlp.614/
%P 12915-12927
Markdown (Informal)
[Stop Rolling the Dice: Active Group Relative Policy Optimization via Intrinsic Metacognition](https://aclanthology.org/2026.findings-emnlp.614/) (Gong et al., Findings 2026)
ACL