@inproceedings{yang-etal-2026-consistency,
title = "Consistency-Calibrated Advantage Estimation for Label-Free Reinforcement Learning",
author = "Yang, Meng and
Zhao, Yongle and
Zhang, Zhiyang and
Jiang, Xiaorui and
Liang, Peirou and
Chen, Furui and
Xie, Yin and
Feng, Ziyong",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2484/",
pages = "56413--56425",
ISBN = "979-8-89176-459-0",
abstract = "Reinforcement learning from unlabeled prompts offers a promising path to improving LLM reasoning without curated supervision or verifiable rewards. However, existing label-free methods typically rely on either intrinsic self-certainty or extrinsic answer agreement, both of which can become unreliable in hard reasoning regimes: confident trajectories can be wrong, while majority answers can drift toward stable yet incorrect attractors. We propose CCAE, a verifier-free and parameter-free advantage estimator that formulates label-free GRPO as a multi-signal advantage estimation problem. CCAE decomposes the advantage into a base self-certainty term, an intrinsic evidence factor, and an extrinsic agreement factor, with their interaction adaptively calibrated by a group-level proxy for agreement reliability, without introducing any reward model, verifier, or trainable parameters. On MATH-Train, CCAE improves over the strongest unlabeled baseline by 5.0 and 4.3 points on Qwen3-4B/8B-Base, respectively, matching supervised GRPO. On the harder DAPO-14k-MATH, CCAE remains stable while prior unlabeled methods collapse, outperforming their best pre-collapse checkpoints by up to 11.1 points. Results on Natural Reasoning further suggest that CCAE remains effective beyond math-only training. Code is available at: [https://github.com/tifing/CCAE](https://github.com/tifing/CCAE)."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yang-etal-2026-consistency">
<titleInfo>
<title>Consistency-Calibrated Advantage Estimation for Label-Free Reinforcement Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Meng</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yongle</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhiyang</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaorui</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peirou</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Furui</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yin</namePart>
<namePart type="family">Xie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziyong</namePart>
<namePart type="family">Feng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Reinforcement learning from unlabeled prompts offers a promising path to improving LLM reasoning without curated supervision or verifiable rewards. However, existing label-free methods typically rely on either intrinsic self-certainty or extrinsic answer agreement, both of which can become unreliable in hard reasoning regimes: confident trajectories can be wrong, while majority answers can drift toward stable yet incorrect attractors. We propose CCAE, a verifier-free and parameter-free advantage estimator that formulates label-free GRPO as a multi-signal advantage estimation problem. CCAE decomposes the advantage into a base self-certainty term, an intrinsic evidence factor, and an extrinsic agreement factor, with their interaction adaptively calibrated by a group-level proxy for agreement reliability, without introducing any reward model, verifier, or trainable parameters. On MATH-Train, CCAE improves over the strongest unlabeled baseline by 5.0 and 4.3 points on Qwen3-4B/8B-Base, respectively, matching supervised GRPO. On the harder DAPO-14k-MATH, CCAE remains stable while prior unlabeled methods collapse, outperforming their best pre-collapse checkpoints by up to 11.1 points. Results on Natural Reasoning further suggest that CCAE remains effective beyond math-only training. Code is available at: [https://github.com/tifing/CCAE](https://github.com/tifing/CCAE).</abstract>
<identifier type="citekey">yang-etal-2026-consistency</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2484/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>56413</start>
<end>56425</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Consistency-Calibrated Advantage Estimation for Label-Free Reinforcement Learning
%A Yang, Meng
%A Zhao, Yongle
%A Zhang, Zhiyang
%A Jiang, Xiaorui
%A Liang, Peirou
%A Chen, Furui
%A Xie, Yin
%A Feng, Ziyong
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F yang-etal-2026-consistency
%X Reinforcement learning from unlabeled prompts offers a promising path to improving LLM reasoning without curated supervision or verifiable rewards. However, existing label-free methods typically rely on either intrinsic self-certainty or extrinsic answer agreement, both of which can become unreliable in hard reasoning regimes: confident trajectories can be wrong, while majority answers can drift toward stable yet incorrect attractors. We propose CCAE, a verifier-free and parameter-free advantage estimator that formulates label-free GRPO as a multi-signal advantage estimation problem. CCAE decomposes the advantage into a base self-certainty term, an intrinsic evidence factor, and an extrinsic agreement factor, with their interaction adaptively calibrated by a group-level proxy for agreement reliability, without introducing any reward model, verifier, or trainable parameters. On MATH-Train, CCAE improves over the strongest unlabeled baseline by 5.0 and 4.3 points on Qwen3-4B/8B-Base, respectively, matching supervised GRPO. On the harder DAPO-14k-MATH, CCAE remains stable while prior unlabeled methods collapse, outperforming their best pre-collapse checkpoints by up to 11.1 points. Results on Natural Reasoning further suggest that CCAE remains effective beyond math-only training. Code is available at: [https://github.com/tifing/CCAE](https://github.com/tifing/CCAE).
%U https://aclanthology.org/2026.emnlp-main.2484/
%P 56413-56425
Markdown (Informal)
[Consistency-Calibrated Advantage Estimation for Label-Free Reinforcement Learning](https://aclanthology.org/2026.emnlp-main.2484/) (Yang et al., EMNLP 2026)
ACL
- Meng Yang, Yongle Zhao, Zhiyang Zhang, Xiaorui Jiang, Peirou Liang, Furui Chen, Yin Xie, and Ziyong Feng. 2026. Consistency-Calibrated Advantage Estimation for Label-Free Reinforcement Learning. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 56413–56425, Budapest, Hungary. Association for Computational Linguistics.