@inproceedings{mei-etal-2026-learning,
title = "Learning to Think in Blocks: A Prior-Guided Reinforcement Learning Framework for Deep Research",
author = "Mei, Wangyi and
Xu, Ganlin and
Zhang, Zhoujia and
Liang, Jiaqing and
Lu, Weijia and
Huang, Jinhui and
Yang, Chen and
Zhang, Xiaodong and
Yang, Zhifei and
Ma, Xiaofeng and
Yang, Deqing",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1067/",
pages = "22566--22584",
ISBN = "979-8-89176-462-0",
abstract = "Reinforcement learning (RL) for multi-turn deep research agents suffers from a granularity mismatch: token-level methods incur high variance over long reasoning horizons, while sequence-level methods ignore the internal decision structure of multi-turn interactions. We introduce Block-wise Geometric Policy Optimization (BGPO), which treats each assistant turn{---}packaging reasoning and tool invocation into one semantically coherent unit{---}as an optimization-level macro-action and aggregates block-averaged log-importance ratios with equal weights across blocks. We prove that block-equal weighting minimizes a covariance-robust upper bound on ratio variance under weak inter-block coupling, with the approximation gap relative to sequence-length weighting bounded by block-length imbalance. To improve exploration efficiency, we expose human-defined rewrite operators{---}including decomposition, keyword extraction, synonym replacement, and HyDE{---}as a prompt-level exploration prior during RL rollouts. On seven knowledge-intensive QA benchmarks, our 7B system achieves an average judge score of 0.715 {\ensuremath{\pm}} 0.010 over three training seeds, compared with 0.664 for Search-R1-7B. A controlled fixed-prior attribution study further confirms that block-level optimization independently improves answer quality, achieving 0.527/0.395 F1/EM compared with 0.482/0.350 for GRPO. Our code is publicly available at https://github.com/meiotoufa/Learning-to-Think-in-Blocks."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mei-etal-2026-learning">
<titleInfo>
<title>Learning to Think in Blocks: A Prior-Guided Reinforcement Learning Framework for Deep Research</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wangyi</namePart>
<namePart type="family">Mei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ganlin</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhoujia</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiaqing</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weijia</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jinhui</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chen</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaodong</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhifei</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaofeng</namePart>
<namePart type="family">Ma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Deqing</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Reinforcement learning (RL) for multi-turn deep research agents suffers from a granularity mismatch: token-level methods incur high variance over long reasoning horizons, while sequence-level methods ignore the internal decision structure of multi-turn interactions. We introduce Block-wise Geometric Policy Optimization (BGPO), which treats each assistant turn—packaging reasoning and tool invocation into one semantically coherent unit—as an optimization-level macro-action and aggregates block-averaged log-importance ratios with equal weights across blocks. We prove that block-equal weighting minimizes a covariance-robust upper bound on ratio variance under weak inter-block coupling, with the approximation gap relative to sequence-length weighting bounded by block-length imbalance. To improve exploration efficiency, we expose human-defined rewrite operators—including decomposition, keyword extraction, synonym replacement, and HyDE—as a prompt-level exploration prior during RL rollouts. On seven knowledge-intensive QA benchmarks, our 7B system achieves an average judge score of 0.715 \ensuremath\pm 0.010 over three training seeds, compared with 0.664 for Search-R1-7B. A controlled fixed-prior attribution study further confirms that block-level optimization independently improves answer quality, achieving 0.527/0.395 F1/EM compared with 0.482/0.350 for GRPO. Our code is publicly available at https://github.com/meiotoufa/Learning-to-Think-in-Blocks.</abstract>
<identifier type="citekey">mei-etal-2026-learning</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1067/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>22566</start>
<end>22584</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Learning to Think in Blocks: A Prior-Guided Reinforcement Learning Framework for Deep Research
%A Mei, Wangyi
%A Xu, Ganlin
%A Zhang, Zhoujia
%A Liang, Jiaqing
%A Lu, Weijia
%A Huang, Jinhui
%A Yang, Chen
%A Zhang, Xiaodong
%A Yang, Zhifei
%A Ma, Xiaofeng
%A Yang, Deqing
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F mei-etal-2026-learning
%X Reinforcement learning (RL) for multi-turn deep research agents suffers from a granularity mismatch: token-level methods incur high variance over long reasoning horizons, while sequence-level methods ignore the internal decision structure of multi-turn interactions. We introduce Block-wise Geometric Policy Optimization (BGPO), which treats each assistant turn—packaging reasoning and tool invocation into one semantically coherent unit—as an optimization-level macro-action and aggregates block-averaged log-importance ratios with equal weights across blocks. We prove that block-equal weighting minimizes a covariance-robust upper bound on ratio variance under weak inter-block coupling, with the approximation gap relative to sequence-length weighting bounded by block-length imbalance. To improve exploration efficiency, we expose human-defined rewrite operators—including decomposition, keyword extraction, synonym replacement, and HyDE—as a prompt-level exploration prior during RL rollouts. On seven knowledge-intensive QA benchmarks, our 7B system achieves an average judge score of 0.715 \ensuremath\pm 0.010 over three training seeds, compared with 0.664 for Search-R1-7B. A controlled fixed-prior attribution study further confirms that block-level optimization independently improves answer quality, achieving 0.527/0.395 F1/EM compared with 0.482/0.350 for GRPO. Our code is publicly available at https://github.com/meiotoufa/Learning-to-Think-in-Blocks.
%U https://aclanthology.org/2026.findings-emnlp.1067/
%P 22566-22584
Markdown (Informal)
[Learning to Think in Blocks: A Prior-Guided Reinforcement Learning Framework for Deep Research](https://aclanthology.org/2026.findings-emnlp.1067/) (Mei et al., Findings 2026)
ACL
- Wangyi Mei, Ganlin Xu, Zhoujia Zhang, Jiaqing Liang, Weijia Lu, Jinhui Huang, Chen Yang, Xiaodong Zhang, Zhifei Yang, Xiaofeng Ma, and Deqing Yang. 2026. Learning to Think in Blocks: A Prior-Guided Reinforcement Learning Framework for Deep Research. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 22566–22584, Budapest, Hungary. Association for Computational Linguistics.