@inproceedings{kokane-2026-much,
title = "How Much Supervised Fine-Tuning Is Enough? Dissecting the Impact of {SFT} Extent on Downstream Reinforcement Learning Performance",
author = "Kokane, Shirley",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.128/",
pages = "2960--2971",
ISBN = "979-8-89176-459-0",
abstract = "Large Language Models (LLMs) are typically adapted to downstream tasks through Supervised Fine-Tuning (SFT), where Cross Entropy (CE) loss is used to maximize the likelihood of ground-truth tokens. While effective for aligning with observed data and establishing strong domain grounding, SFT alone often limits the model{'}s ability to explore alternative plausible outputs, potentially narrowing its decision space. Conversely, Reinforcement Learning (RL) enables reward-driven exploration and refinement but relies heavily on a well-aligned initialization to achieve stable and sample-efficient training. In real-world deployments, practitioners frequently grapple with the question: How much SFT is sufficient, and when should RL take over? This work investigates the complementary trade-off between SFT and RL approaches such as GRPO, focusing on how both stages jointly influence model exploration and exploitation. We provide a quantitative analysis across multiple domains to understand how varying the extent of SFT influences a model{'}s exploratory behavior and downstream RL effectiveness. Our findings aim to guide in determining optimal SFT checkpoints (CKPTs) that balance domain alignment with exploration capacity, thereby maximizing final domain performance while minimizing overfitting after the joint post-training. Our data and code are open source at https://github.com/Shirley-Kokane/sft2rl."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kokane-2026-much">
<titleInfo>
<title>How Much Supervised Fine-Tuning Is Enough? Dissecting the Impact of SFT Extent on Downstream Reinforcement Learning Performance</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shirley</namePart>
<namePart type="family">Kokane</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large Language Models (LLMs) are typically adapted to downstream tasks through Supervised Fine-Tuning (SFT), where Cross Entropy (CE) loss is used to maximize the likelihood of ground-truth tokens. While effective for aligning with observed data and establishing strong domain grounding, SFT alone often limits the model’s ability to explore alternative plausible outputs, potentially narrowing its decision space. Conversely, Reinforcement Learning (RL) enables reward-driven exploration and refinement but relies heavily on a well-aligned initialization to achieve stable and sample-efficient training. In real-world deployments, practitioners frequently grapple with the question: How much SFT is sufficient, and when should RL take over? This work investigates the complementary trade-off between SFT and RL approaches such as GRPO, focusing on how both stages jointly influence model exploration and exploitation. We provide a quantitative analysis across multiple domains to understand how varying the extent of SFT influences a model’s exploratory behavior and downstream RL effectiveness. Our findings aim to guide in determining optimal SFT checkpoints (CKPTs) that balance domain alignment with exploration capacity, thereby maximizing final domain performance while minimizing overfitting after the joint post-training. Our data and code are open source at https://github.com/Shirley-Kokane/sft2rl.</abstract>
<identifier type="citekey">kokane-2026-much</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.128/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>2960</start>
<end>2971</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T How Much Supervised Fine-Tuning Is Enough? Dissecting the Impact of SFT Extent on Downstream Reinforcement Learning Performance
%A Kokane, Shirley
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F kokane-2026-much
%X Large Language Models (LLMs) are typically adapted to downstream tasks through Supervised Fine-Tuning (SFT), where Cross Entropy (CE) loss is used to maximize the likelihood of ground-truth tokens. While effective for aligning with observed data and establishing strong domain grounding, SFT alone often limits the model’s ability to explore alternative plausible outputs, potentially narrowing its decision space. Conversely, Reinforcement Learning (RL) enables reward-driven exploration and refinement but relies heavily on a well-aligned initialization to achieve stable and sample-efficient training. In real-world deployments, practitioners frequently grapple with the question: How much SFT is sufficient, and when should RL take over? This work investigates the complementary trade-off between SFT and RL approaches such as GRPO, focusing on how both stages jointly influence model exploration and exploitation. We provide a quantitative analysis across multiple domains to understand how varying the extent of SFT influences a model’s exploratory behavior and downstream RL effectiveness. Our findings aim to guide in determining optimal SFT checkpoints (CKPTs) that balance domain alignment with exploration capacity, thereby maximizing final domain performance while minimizing overfitting after the joint post-training. Our data and code are open source at https://github.com/Shirley-Kokane/sft2rl.
%U https://aclanthology.org/2026.emnlp-main.128/
%P 2960-2971
Markdown (Informal)
[How Much Supervised Fine-Tuning Is Enough? Dissecting the Impact of SFT Extent on Downstream Reinforcement Learning Performance](https://aclanthology.org/2026.emnlp-main.128/) (Kokane, EMNLP 2026)
ACL