@inproceedings{li-etal-2026-densepath,
title = "{D}ense{P}ath {REINFORCE}: Lightweight Policy Improvement from {SFT}-Induced Token-Level Reward Signals",
author = "Li, Jiangnan and
Vu, Thuy-Trang and
Abbasnejad, Ehsan and
Haffari, Gholamreza",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.267/",
pages = "5924--5941",
ISBN = "979-8-89176-439-2",
abstract = "Supervised fine-tuning (SFT) is typically viewed as behavior cloning on expert demonstrations. Recent methods recover richer supervision from demonstrations through self-play preference synthesis or bilevel inverse reinforcement learning, yet these require additional optimization stages or separately trained reward models. Motivated by the connection between SFT and inverse soft-Q learning, we propose Dense-Path REINFORCE (DPR), a lightweight post-SFT policy-improvement method that extracts a dense reward signal directly from the SFT training trajectory. We construct a token-level reward from the log-probability gap between the final SFT policy and an intermediate checkpoint, capturing the incremental likelihood gain acquired during training while mitigating the length and early-EOS biases inherent in raw log-probability rewards. DPR reuses the same demonstrations and prompts as SFT, requires no preference data, no external teacher, and no separately trained reward model. Across four pretrained backbones and four instruction-following evaluations, DPR obtains 55.2-66.8{\%} win rates over SFT, improves MT-Bench by 0.22-0.46 points, and matches or exceeds the best demonstration-only baseline in 15 of 16 settings."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-etal-2026-densepath">
<titleInfo>
<title>DensePath REINFORCE: Lightweight Policy Improvement from SFT-Induced Token-Level Reward Signals</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jiangnan</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thuy-Trang</namePart>
<namePart type="family">Vu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ehsan</namePart>
<namePart type="family">Abbasnejad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Supervised fine-tuning (SFT) is typically viewed as behavior cloning on expert demonstrations. Recent methods recover richer supervision from demonstrations through self-play preference synthesis or bilevel inverse reinforcement learning, yet these require additional optimization stages or separately trained reward models. Motivated by the connection between SFT and inverse soft-Q learning, we propose Dense-Path REINFORCE (DPR), a lightweight post-SFT policy-improvement method that extracts a dense reward signal directly from the SFT training trajectory. We construct a token-level reward from the log-probability gap between the final SFT policy and an intermediate checkpoint, capturing the incremental likelihood gain acquired during training while mitigating the length and early-EOS biases inherent in raw log-probability rewards. DPR reuses the same demonstrations and prompts as SFT, requires no preference data, no external teacher, and no separately trained reward model. Across four pretrained backbones and four instruction-following evaluations, DPR obtains 55.2-66.8% win rates over SFT, improves MT-Bench by 0.22-0.46 points, and matches or exceeds the best demonstration-only baseline in 15 of 16 settings.</abstract>
<identifier type="citekey">li-etal-2026-densepath</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.267/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>5924</start>
<end>5941</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T DensePath REINFORCE: Lightweight Policy Improvement from SFT-Induced Token-Level Reward Signals
%A Li, Jiangnan
%A Vu, Thuy-Trang
%A Abbasnejad, Ehsan
%A Haffari, Gholamreza
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F li-etal-2026-densepath
%X Supervised fine-tuning (SFT) is typically viewed as behavior cloning on expert demonstrations. Recent methods recover richer supervision from demonstrations through self-play preference synthesis or bilevel inverse reinforcement learning, yet these require additional optimization stages or separately trained reward models. Motivated by the connection between SFT and inverse soft-Q learning, we propose Dense-Path REINFORCE (DPR), a lightweight post-SFT policy-improvement method that extracts a dense reward signal directly from the SFT training trajectory. We construct a token-level reward from the log-probability gap between the final SFT policy and an intermediate checkpoint, capturing the incremental likelihood gain acquired during training while mitigating the length and early-EOS biases inherent in raw log-probability rewards. DPR reuses the same demonstrations and prompts as SFT, requires no preference data, no external teacher, and no separately trained reward model. Across four pretrained backbones and four instruction-following evaluations, DPR obtains 55.2-66.8% win rates over SFT, improves MT-Bench by 0.22-0.46 points, and matches or exceeds the best demonstration-only baseline in 15 of 16 settings.
%U https://aclanthology.org/2026.aacl-long.267/
%P 5924-5941
Markdown (Informal)
[DensePath REINFORCE: Lightweight Policy Improvement from SFT-Induced Token-Level Reward Signals](https://aclanthology.org/2026.aacl-long.267/) (Li et al., AACL-IJCNLP 2026)
ACL