@inproceedings{chegini-feizi-2026-training,
title = "Training-Free Chunk-Level Guided Generation for Mathematical Reasoning",
author = "Chegini, Atoosa and
Feizi, Soheil",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2591/",
pages = "58859--58870",
ISBN = "979-8-89176-459-0",
abstract = "Selecting the best response from multiple small-model samples using a stronger scorer is a simple inference-time strategy, but fails in practice because the small model has already committed to diverging reasoning paths that no scorer can correct; PRM guided search avoids this by scoring candidate continuations during generation, but requires a reward model trained with step-level labels.\par We propose Chunk-Level Guided Generation, a training-free alternative that uses an off-the-shelf large language model as a process scorer: a small model samples $k$ fixed-length candidate chunks at each step, while the larger language model scores the candidates using likelihoods, without generating any text. The selected chunk is committed before the next step, steering generation before errors can propagate.\par We instantiate this framework with two selection rules: Likelihood-Guided Selection (LGS), which selects the chunk with the highest length-normalized large-model log-probability, and Contrastive-Guided Selection (CGS), which subtracts the small model{'}s log-probability to favor chunks where the large model{'}s preference diverges from the small model{'}s. We show that scoring variable-length reasoning steps with large-model likelihoods is unreliable due to a systematic length bias that persists even after length normalization, and that fixed-length chunks avoid this confound.\par On GSM8K, MATH, Minerva Math, AMC23, and AIME24 with Qwen2.5-1.5B guided by Qwen2.5-32B and Llama-3.2-1B guided by Llama-3.1-70B, CGS outperforms majority voting by up to 28pp (e.g., 83.9{\%} vs. 56.0{\%} on GSM8K with the Llama pair at $k{=}32$) and, under matched guidance budgets, matches or outperforms Qwen2.5-Math-PRM-72B guided search on most benchmarks without any reward-model training (e.g., 39.0{\%} vs. 32.4{\%} on Minerva Math with the Qwen pair at $k{=}32$). With Qwen2.5-7B guided by Qwen2.5-72B, CGS reaches 81.8{\%} on MATH and 63.6{\%} on Minerva Math at $k{=}16$, surpassing majority voting by 4{--}6 pp.\par Finally, we show that Chunk-Level Guided Generation produces substantially shorter reasoning traces than PRM guided search, with PRM responses up to 80{\%} longer on GSM8K."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chegini-feizi-2026-training">
<titleInfo>
<title>Training-Free Chunk-Level Guided Generation for Mathematical Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Atoosa</namePart>
<namePart type="family">Chegini</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Soheil</namePart>
<namePart type="family">Feizi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Selecting the best response from multiple small-model samples using a stronger scorer is a simple inference-time strategy, but fails in practice because the small model has already committed to diverging reasoning paths that no scorer can correct; PRM guided search avoids this by scoring candidate continuations during generation, but requires a reward model trained with step-level labels.\par We propose Chunk-Level Guided Generation, a training-free alternative that uses an off-the-shelf large language model as a process scorer: a small model samples k fixed-length candidate chunks at each step, while the larger language model scores the candidates using likelihoods, without generating any text. The selected chunk is committed before the next step, steering generation before errors can propagate.\par We instantiate this framework with two selection rules: Likelihood-Guided Selection (LGS), which selects the chunk with the highest length-normalized large-model log-probability, and Contrastive-Guided Selection (CGS), which subtracts the small model’s log-probability to favor chunks where the large model’s preference diverges from the small model’s. We show that scoring variable-length reasoning steps with large-model likelihoods is unreliable due to a systematic length bias that persists even after length normalization, and that fixed-length chunks avoid this confound.\par On GSM8K, MATH, Minerva Math, AMC23, and AIME24 with Qwen2.5-1.5B guided by Qwen2.5-32B and Llama-3.2-1B guided by Llama-3.1-70B, CGS outperforms majority voting by up to 28pp (e.g., 83.9% vs. 56.0% on GSM8K with the Llama pair at k=32) and, under matched guidance budgets, matches or outperforms Qwen2.5-Math-PRM-72B guided search on most benchmarks without any reward-model training (e.g., 39.0% vs. 32.4% on Minerva Math with the Qwen pair at k=32). With Qwen2.5-7B guided by Qwen2.5-72B, CGS reaches 81.8% on MATH and 63.6% on Minerva Math at k=16, surpassing majority voting by 4–6 pp.\par Finally, we show that Chunk-Level Guided Generation produces substantially shorter reasoning traces than PRM guided search, with PRM responses up to 80% longer on GSM8K.</abstract>
<identifier type="citekey">chegini-feizi-2026-training</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2591/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>58859</start>
<end>58870</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Training-Free Chunk-Level Guided Generation for Mathematical Reasoning
%A Chegini, Atoosa
%A Feizi, Soheil
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F chegini-feizi-2026-training
%X Selecting the best response from multiple small-model samples using a stronger scorer is a simple inference-time strategy, but fails in practice because the small model has already committed to diverging reasoning paths that no scorer can correct; PRM guided search avoids this by scoring candidate continuations during generation, but requires a reward model trained with step-level labels.\par We propose Chunk-Level Guided Generation, a training-free alternative that uses an off-the-shelf large language model as a process scorer: a small model samples k fixed-length candidate chunks at each step, while the larger language model scores the candidates using likelihoods, without generating any text. The selected chunk is committed before the next step, steering generation before errors can propagate.\par We instantiate this framework with two selection rules: Likelihood-Guided Selection (LGS), which selects the chunk with the highest length-normalized large-model log-probability, and Contrastive-Guided Selection (CGS), which subtracts the small model’s log-probability to favor chunks where the large model’s preference diverges from the small model’s. We show that scoring variable-length reasoning steps with large-model likelihoods is unreliable due to a systematic length bias that persists even after length normalization, and that fixed-length chunks avoid this confound.\par On GSM8K, MATH, Minerva Math, AMC23, and AIME24 with Qwen2.5-1.5B guided by Qwen2.5-32B and Llama-3.2-1B guided by Llama-3.1-70B, CGS outperforms majority voting by up to 28pp (e.g., 83.9% vs. 56.0% on GSM8K with the Llama pair at k=32) and, under matched guidance budgets, matches or outperforms Qwen2.5-Math-PRM-72B guided search on most benchmarks without any reward-model training (e.g., 39.0% vs. 32.4% on Minerva Math with the Qwen pair at k=32). With Qwen2.5-7B guided by Qwen2.5-72B, CGS reaches 81.8% on MATH and 63.6% on Minerva Math at k=16, surpassing majority voting by 4–6 pp.\par Finally, we show that Chunk-Level Guided Generation produces substantially shorter reasoning traces than PRM guided search, with PRM responses up to 80% longer on GSM8K.
%U https://aclanthology.org/2026.emnlp-main.2591/
%P 58859-58870
Markdown (Informal)
[Training-Free Chunk-Level Guided Generation for Mathematical Reasoning](https://aclanthology.org/2026.emnlp-main.2591/) (Chegini & Feizi, EMNLP 2026)
ACL