@inproceedings{yuan-etal-2026-mitigating,
title = "Mitigating Forgetting Between Supervised and Reinforcement Learning Yields Stronger Reasoners",
author = "Yuan, Xiangchi and
Chen, Xiang and
Yu, Tong and
Shi, Dachuan and
Jin, Can and
Lee, Wenke and
Mitra, Saayan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.730/",
pages = "16948--16971",
ISBN = "979-8-89176-459-0",
abstract = "State-of-the-art (SOTA) reasoning post-training pipelines increasingly combine supervised fine-tuning (SFT) with reinforcement learning (RL) to balance external knowledge injection and self-exploration. In this work, we identify a critical but underexplored failure mode of these SFT+RL pipelines: catastrophic forgetting. To analyze the cause, we reveal a fundamental asymmetry between SFT and RL. SFT typically induces much larger, redundant parameter updates than RL, while RL{'}s updates are smaller and more parsimonious. Therefore, SFT updates can easily overwrite RL{'}s small non-robust updates, causing the forgetting of the knowledge learned by RL. To address this issue, we propose MIFO (MItigating FOrgetting between SFT and RL), a plug-and-play framework that integrates SFT into RL while explicitly constraining SFT-induced updates. MIFO mitigates forgetting through two complementary designs: (i) Data selection, which interleaves RL and SFT by selecting challenging samples based on rollout accuracy and optimizing only high-entropy tokens for SFT; and (ii) parameter freezing, which identifies RL-critical parameters and temporarily freezes them during SFT to prevent overwrite. Beyond stability, MIFO is data- and response-efficient and remains agnostic to the underlying SFT or RL algorithm. Empirically, MIFO achieves SOTA reasoning performance using only 1.5{\%} of the SFT data and 20.4{\%} of the RL data required by prior SOTA methods, while producing reasoning traces up to 10$\times$ shorter than the standard SFT$\to$RL pipeline."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yuan-etal-2026-mitigating">
<titleInfo>
<title>Mitigating Forgetting Between Supervised and Reinforcement Learning Yields Stronger Reasoners</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xiangchi</namePart>
<namePart type="family">Yuan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiang</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dachuan</namePart>
<namePart type="family">Shi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Can</namePart>
<namePart type="family">Jin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenke</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Saayan</namePart>
<namePart type="family">Mitra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>State-of-the-art (SOTA) reasoning post-training pipelines increasingly combine supervised fine-tuning (SFT) with reinforcement learning (RL) to balance external knowledge injection and self-exploration. In this work, we identify a critical but underexplored failure mode of these SFT+RL pipelines: catastrophic forgetting. To analyze the cause, we reveal a fundamental asymmetry between SFT and RL. SFT typically induces much larger, redundant parameter updates than RL, while RL’s updates are smaller and more parsimonious. Therefore, SFT updates can easily overwrite RL’s small non-robust updates, causing the forgetting of the knowledge learned by RL. To address this issue, we propose MIFO (MItigating FOrgetting between SFT and RL), a plug-and-play framework that integrates SFT into RL while explicitly constraining SFT-induced updates. MIFO mitigates forgetting through two complementary designs: (i) Data selection, which interleaves RL and SFT by selecting challenging samples based on rollout accuracy and optimizing only high-entropy tokens for SFT; and (ii) parameter freezing, which identifies RL-critical parameters and temporarily freezes them during SFT to prevent overwrite. Beyond stability, MIFO is data- and response-efficient and remains agnostic to the underlying SFT or RL algorithm. Empirically, MIFO achieves SOTA reasoning performance using only 1.5% of the SFT data and 20.4% of the RL data required by prior SOTA methods, while producing reasoning traces up to 10\times shorter than the standard SFTRL pipeline.</abstract>
<identifier type="citekey">yuan-etal-2026-mitigating</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.730/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>16948</start>
<end>16971</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Mitigating Forgetting Between Supervised and Reinforcement Learning Yields Stronger Reasoners
%A Yuan, Xiangchi
%A Chen, Xiang
%A Yu, Tong
%A Shi, Dachuan
%A Jin, Can
%A Lee, Wenke
%A Mitra, Saayan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F yuan-etal-2026-mitigating
%X State-of-the-art (SOTA) reasoning post-training pipelines increasingly combine supervised fine-tuning (SFT) with reinforcement learning (RL) to balance external knowledge injection and self-exploration. In this work, we identify a critical but underexplored failure mode of these SFT+RL pipelines: catastrophic forgetting. To analyze the cause, we reveal a fundamental asymmetry between SFT and RL. SFT typically induces much larger, redundant parameter updates than RL, while RL’s updates are smaller and more parsimonious. Therefore, SFT updates can easily overwrite RL’s small non-robust updates, causing the forgetting of the knowledge learned by RL. To address this issue, we propose MIFO (MItigating FOrgetting between SFT and RL), a plug-and-play framework that integrates SFT into RL while explicitly constraining SFT-induced updates. MIFO mitigates forgetting through two complementary designs: (i) Data selection, which interleaves RL and SFT by selecting challenging samples based on rollout accuracy and optimizing only high-entropy tokens for SFT; and (ii) parameter freezing, which identifies RL-critical parameters and temporarily freezes them during SFT to prevent overwrite. Beyond stability, MIFO is data- and response-efficient and remains agnostic to the underlying SFT or RL algorithm. Empirically, MIFO achieves SOTA reasoning performance using only 1.5% of the SFT data and 20.4% of the RL data required by prior SOTA methods, while producing reasoning traces up to 10\times shorter than the standard SFTRL pipeline.
%U https://aclanthology.org/2026.emnlp-main.730/
%P 16948-16971
Markdown (Informal)
[Mitigating Forgetting Between Supervised and Reinforcement Learning Yields Stronger Reasoners](https://aclanthology.org/2026.emnlp-main.730/) (Yuan et al., EMNLP 2026)
ACL