@inproceedings{sun-etal-2026-base,
title = "From Base Rollouts to {RL} Reasoning: A Budgeted Search Perspective",
author = "Sun, Wenhe and
Wang, Cunxiang and
Yao, Zijun and
Cao, Yixin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1593/",
pages = "33908--33931",
ISBN = "979-8-89176-462-0",
abstract = "Reinforcement learning with verifiable rewards (RLVR) improves language-model reasoning, but how these gains relate to inference-time decoding and search remains unclear. Does RL produce reasoning behavior the base model does not have, or does it shift the rollout distribution toward trajectories the base model can already reach but rarely samples? We study this behaviorally with a Unified Decoding Framework (UDF), which expresses token-level sampling, beam-like search, tree search and sequence-level resampling as executable policies over a shared budgeted operating space. Operating points are scored after generation with pass@$k$, self-consistency, best-of-$N$ and first-finish success, so the generation policy stays separate from the evaluation metric.\par Using paired Base/RL checkpoints from SimpleRL-Zoo, we ask whether an RL default-policy curve can be approximated by a structured path of Base operating points $(\pi, N_{\mathrm{Base}})$ rather than by post-hoc per-budget matching. On Math500, AIME, GPQA and IFEval, the pass@$k$ recovery path follows a Budgeted Operating-Point Transition Rule (BOPTR), $N_{\mathrm{Base}} \approx \alpha N_{\mathrm{RL}}^{\beta}$, with benchmark-conditioned exponents. On Qwen2.5-7B, BOPTR gives the lowest transfer error among the non-oracle rules we test, 3.41 pp. Across models the rule family remains informative, but its parameters and selected policies are model-specific: Qwen models trained on the same difficulty split share operating-point structure, while cross-family models and different training distributions mark the limits of direct transfer.\par These results support a qualified internalized-search reading: under the recipe we test, much of the measured RL gain corresponds to a change in sampling efficiency, toward operating points the base model can already reach under search. We treat the benchmark-wise scaling patterns as descriptive of this recipe and cohort, report the cases in which they break down, and use UDF and BOPTR as behavioral diagnostics rather than as evidence of parameter-level equivalence or as general predictors of post-RL performance."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sun-etal-2026-base">
<titleInfo>
<title>From Base Rollouts to RL Reasoning: A Budgeted Search Perspective</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wenhe</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cunxiang</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zijun</namePart>
<namePart type="family">Yao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yixin</namePart>
<namePart type="family">Cao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Reinforcement learning with verifiable rewards (RLVR) improves language-model reasoning, but how these gains relate to inference-time decoding and search remains unclear. Does RL produce reasoning behavior the base model does not have, or does it shift the rollout distribution toward trajectories the base model can already reach but rarely samples? We study this behaviorally with a Unified Decoding Framework (UDF), which expresses token-level sampling, beam-like search, tree search and sequence-level resampling as executable policies over a shared budgeted operating space. Operating points are scored after generation with pass@k, self-consistency, best-of-N and first-finish success, so the generation policy stays separate from the evaluation metric.\par Using paired Base/RL checkpoints from SimpleRL-Zoo, we ask whether an RL default-policy curve can be approximated by a structured path of Base operating points (π, N_Base) rather than by post-hoc per-budget matching. On Math500, AIME, GPQA and IFEval, the pass@k recovery path follows a Budgeted Operating-Point Transition Rule (BOPTR), N_Base \approx α N_RL^β, with benchmark-conditioned exponents. On Qwen2.5-7B, BOPTR gives the lowest transfer error among the non-oracle rules we test, 3.41 pp. Across models the rule family remains informative, but its parameters and selected policies are model-specific: Qwen models trained on the same difficulty split share operating-point structure, while cross-family models and different training distributions mark the limits of direct transfer.\par These results support a qualified internalized-search reading: under the recipe we test, much of the measured RL gain corresponds to a change in sampling efficiency, toward operating points the base model can already reach under search. We treat the benchmark-wise scaling patterns as descriptive of this recipe and cohort, report the cases in which they break down, and use UDF and BOPTR as behavioral diagnostics rather than as evidence of parameter-level equivalence or as general predictors of post-RL performance.</abstract>
<identifier type="citekey">sun-etal-2026-base</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1593/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>33908</start>
<end>33931</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T From Base Rollouts to RL Reasoning: A Budgeted Search Perspective
%A Sun, Wenhe
%A Wang, Cunxiang
%A Yao, Zijun
%A Cao, Yixin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F sun-etal-2026-base
%X Reinforcement learning with verifiable rewards (RLVR) improves language-model reasoning, but how these gains relate to inference-time decoding and search remains unclear. Does RL produce reasoning behavior the base model does not have, or does it shift the rollout distribution toward trajectories the base model can already reach but rarely samples? We study this behaviorally with a Unified Decoding Framework (UDF), which expresses token-level sampling, beam-like search, tree search and sequence-level resampling as executable policies over a shared budgeted operating space. Operating points are scored after generation with pass@k, self-consistency, best-of-N and first-finish success, so the generation policy stays separate from the evaluation metric.\par Using paired Base/RL checkpoints from SimpleRL-Zoo, we ask whether an RL default-policy curve can be approximated by a structured path of Base operating points (π, N_Base) rather than by post-hoc per-budget matching. On Math500, AIME, GPQA and IFEval, the pass@k recovery path follows a Budgeted Operating-Point Transition Rule (BOPTR), N_Base \approx α N_RL^β, with benchmark-conditioned exponents. On Qwen2.5-7B, BOPTR gives the lowest transfer error among the non-oracle rules we test, 3.41 pp. Across models the rule family remains informative, but its parameters and selected policies are model-specific: Qwen models trained on the same difficulty split share operating-point structure, while cross-family models and different training distributions mark the limits of direct transfer.\par These results support a qualified internalized-search reading: under the recipe we test, much of the measured RL gain corresponds to a change in sampling efficiency, toward operating points the base model can already reach under search. We treat the benchmark-wise scaling patterns as descriptive of this recipe and cohort, report the cases in which they break down, and use UDF and BOPTR as behavioral diagnostics rather than as evidence of parameter-level equivalence or as general predictors of post-RL performance.
%U https://aclanthology.org/2026.findings-emnlp.1593/
%P 33908-33931
Markdown (Informal)
[From Base Rollouts to RL Reasoning: A Budgeted Search Perspective](https://aclanthology.org/2026.findings-emnlp.1593/) (Sun et al., Findings 2026)
ACL