@inproceedings{sanyal-etal-2026-format,
title = "Format Collapse in Policy Optimization for {LLM} Reasoning",
author = "Sanyal, Debdeep and
Sharma, Aakash Sen and
Deshpande, Saurabh and
Kumar, Dhruv and
Mandal, Murari",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1706/",
pages = "38849--38869",
ISBN = "979-8-89176-459-0",
abstract = "Policy optimization (PO) is widely used to post-train language models for reasoning, often with a rigid {\ensuremath{<}}think{\ensuremath{>}}{\ensuremath{<}}answer{\ensuremath{>}} format. We study what PO learns when this constraint is relaxed. We evaluate a relaxed reward across six models and four benchmarks, then audit learned formats on GSM8K and controlled variants across three on-policy algorithms. In these audits, accepting either explicit reasoning or a direct answer pushes the policy toward the shortest valid format: an {\ensuremath{<}}answer{\ensuremath{>}}-only response. On GSM8K, the collapsed Qwen-2.5 7B policy produces answer-only outputs in 98.6{\%} of generations and loses 7.1 pass@5 points relative to a strict-format baseline. Controlled rewards show this is not a regex artifact: with nested formats, PO learns easy requirements before harder ones despite a 4{\texttimes} hard-format reward; with mutually exclusive equal-reward formats, the audited Qwen-2.5 7B final policy assigns more than 99{\%} of generations to the easiest format. A sample-gated gradient account fits the pattern: formats receive positive updates only when sampled, so early support can outweigh reward scale. KL-controlled runs show collapse appears only when the policy can move far enough from initialization. The result identifies a practical failure mode for RLVR reward design."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sanyal-etal-2026-format">
<titleInfo>
<title>Format Collapse in Policy Optimization for LLM Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Debdeep</namePart>
<namePart type="family">Sanyal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aakash</namePart>
<namePart type="given">Sen</namePart>
<namePart type="family">Sharma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Saurabh</namePart>
<namePart type="family">Deshpande</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dhruv</namePart>
<namePart type="family">Kumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Murari</namePart>
<namePart type="family">Mandal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Policy optimization (PO) is widely used to post-train language models for reasoning, often with a rigid \ensuremath<think\ensuremath>\ensuremath<answer\ensuremath> format. We study what PO learns when this constraint is relaxed. We evaluate a relaxed reward across six models and four benchmarks, then audit learned formats on GSM8K and controlled variants across three on-policy algorithms. In these audits, accepting either explicit reasoning or a direct answer pushes the policy toward the shortest valid format: an \ensuremath<answer\ensuremath>-only response. On GSM8K, the collapsed Qwen-2.5 7B policy produces answer-only outputs in 98.6% of generations and loses 7.1 pass@5 points relative to a strict-format baseline. Controlled rewards show this is not a regex artifact: with nested formats, PO learns easy requirements before harder ones despite a 4× hard-format reward; with mutually exclusive equal-reward formats, the audited Qwen-2.5 7B final policy assigns more than 99% of generations to the easiest format. A sample-gated gradient account fits the pattern: formats receive positive updates only when sampled, so early support can outweigh reward scale. KL-controlled runs show collapse appears only when the policy can move far enough from initialization. The result identifies a practical failure mode for RLVR reward design.</abstract>
<identifier type="citekey">sanyal-etal-2026-format</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1706/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>38849</start>
<end>38869</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Format Collapse in Policy Optimization for LLM Reasoning
%A Sanyal, Debdeep
%A Sharma, Aakash Sen
%A Deshpande, Saurabh
%A Kumar, Dhruv
%A Mandal, Murari
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F sanyal-etal-2026-format
%X Policy optimization (PO) is widely used to post-train language models for reasoning, often with a rigid \ensuremath<think\ensuremath>\ensuremath<answer\ensuremath> format. We study what PO learns when this constraint is relaxed. We evaluate a relaxed reward across six models and four benchmarks, then audit learned formats on GSM8K and controlled variants across three on-policy algorithms. In these audits, accepting either explicit reasoning or a direct answer pushes the policy toward the shortest valid format: an \ensuremath<answer\ensuremath>-only response. On GSM8K, the collapsed Qwen-2.5 7B policy produces answer-only outputs in 98.6% of generations and loses 7.1 pass@5 points relative to a strict-format baseline. Controlled rewards show this is not a regex artifact: with nested formats, PO learns easy requirements before harder ones despite a 4× hard-format reward; with mutually exclusive equal-reward formats, the audited Qwen-2.5 7B final policy assigns more than 99% of generations to the easiest format. A sample-gated gradient account fits the pattern: formats receive positive updates only when sampled, so early support can outweigh reward scale. KL-controlled runs show collapse appears only when the policy can move far enough from initialization. The result identifies a practical failure mode for RLVR reward design.
%U https://aclanthology.org/2026.emnlp-main.1706/
%P 38849-38869
Markdown (Informal)
[Format Collapse in Policy Optimization for LLM Reasoning](https://aclanthology.org/2026.emnlp-main.1706/) (Sanyal et al., EMNLP 2026)
ACL
- Debdeep Sanyal, Aakash Sen Sharma, Saurabh Deshpande, Dhruv Kumar, and Murari Mandal. 2026. Format Collapse in Policy Optimization for LLM Reasoning. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 38849–38869, Budapest, Hungary. Association for Computational Linguistics.