@inproceedings{hao-etal-2026-flipts,
title = "{F}lip{TS}: Failure-to-Success Mathematical Self-Refinement via Trajectory-Aware Sampling in End-to-End Online {RL}",
author = "Hao, Yaqian and
Hao, Yifan and
Liu, Yankai and
Chao, Fangning and
Cui, Zhaojun and
Zhang, Haiyu and
Bai, Huan and
Meng, Fanyu and
Deng, Chao and
Feng, Junlan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2544/",
pages = "57845--57862",
ISBN = "979-8-89176-459-0",
abstract = "Self-Refinement has emerged as a promising paradigm for improving Large Language Model (LLM) performance by iteratively improving its response at inference time. Existing training methods typically treat refinement as a decoupled task and rely on pre-generated responses, thereby detaching training from the model{'}s actual reasoning trajectories and hindering performance. In this paper, we propose FlipTS (Failure-to-Success Learning for self-refinement with Potential-guided Trajectory-Aware Sampling), an end-to-end reinforcement learning (RL) framework that optimizes the entire self-refinement loop online. We identify a fundamental bottleneck in this end-to-end setting: the sparsity of informative refinement behaviors Failure-to-Success (F2S). FlipTS addresses this by utilizing a non-stationary Bayesian model to estimate data potential and applying trajectory-aware sampling to enrich the training process with valuable refinement signals. Experiments on Qwen3-4B-Instruct, Qwen3-1.7B and LLaMA-3.1-8B-Instruct across challenging mathematical benchmarks demonstrate that FlipTS consistently outperforms both offline and online RL baselines. Notably, it exhibits robust generalization to factuality, scientific reasoning and legal reasoning tasks without domain-specific tuning. Our code will be open-sourced."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hao-etal-2026-flipts">
<titleInfo>
<title>FlipTS: Failure-to-Success Mathematical Self-Refinement via Trajectory-Aware Sampling in End-to-End Online RL</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yaqian</namePart>
<namePart type="family">Hao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yifan</namePart>
<namePart type="family">Hao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yankai</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fangning</namePart>
<namePart type="family">Chao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhaojun</namePart>
<namePart type="family">Cui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haiyu</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Huan</namePart>
<namePart type="family">Bai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fanyu</namePart>
<namePart type="family">Meng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chao</namePart>
<namePart type="family">Deng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junlan</namePart>
<namePart type="family">Feng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Self-Refinement has emerged as a promising paradigm for improving Large Language Model (LLM) performance by iteratively improving its response at inference time. Existing training methods typically treat refinement as a decoupled task and rely on pre-generated responses, thereby detaching training from the model’s actual reasoning trajectories and hindering performance. In this paper, we propose FlipTS (Failure-to-Success Learning for self-refinement with Potential-guided Trajectory-Aware Sampling), an end-to-end reinforcement learning (RL) framework that optimizes the entire self-refinement loop online. We identify a fundamental bottleneck in this end-to-end setting: the sparsity of informative refinement behaviors Failure-to-Success (F2S). FlipTS addresses this by utilizing a non-stationary Bayesian model to estimate data potential and applying trajectory-aware sampling to enrich the training process with valuable refinement signals. Experiments on Qwen3-4B-Instruct, Qwen3-1.7B and LLaMA-3.1-8B-Instruct across challenging mathematical benchmarks demonstrate that FlipTS consistently outperforms both offline and online RL baselines. Notably, it exhibits robust generalization to factuality, scientific reasoning and legal reasoning tasks without domain-specific tuning. Our code will be open-sourced.</abstract>
<identifier type="citekey">hao-etal-2026-flipts</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2544/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>57845</start>
<end>57862</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T FlipTS: Failure-to-Success Mathematical Self-Refinement via Trajectory-Aware Sampling in End-to-End Online RL
%A Hao, Yaqian
%A Hao, Yifan
%A Liu, Yankai
%A Chao, Fangning
%A Cui, Zhaojun
%A Zhang, Haiyu
%A Bai, Huan
%A Meng, Fanyu
%A Deng, Chao
%A Feng, Junlan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F hao-etal-2026-flipts
%X Self-Refinement has emerged as a promising paradigm for improving Large Language Model (LLM) performance by iteratively improving its response at inference time. Existing training methods typically treat refinement as a decoupled task and rely on pre-generated responses, thereby detaching training from the model’s actual reasoning trajectories and hindering performance. In this paper, we propose FlipTS (Failure-to-Success Learning for self-refinement with Potential-guided Trajectory-Aware Sampling), an end-to-end reinforcement learning (RL) framework that optimizes the entire self-refinement loop online. We identify a fundamental bottleneck in this end-to-end setting: the sparsity of informative refinement behaviors Failure-to-Success (F2S). FlipTS addresses this by utilizing a non-stationary Bayesian model to estimate data potential and applying trajectory-aware sampling to enrich the training process with valuable refinement signals. Experiments on Qwen3-4B-Instruct, Qwen3-1.7B and LLaMA-3.1-8B-Instruct across challenging mathematical benchmarks demonstrate that FlipTS consistently outperforms both offline and online RL baselines. Notably, it exhibits robust generalization to factuality, scientific reasoning and legal reasoning tasks without domain-specific tuning. Our code will be open-sourced.
%U https://aclanthology.org/2026.emnlp-main.2544/
%P 57845-57862
Markdown (Informal)
[FlipTS: Failure-to-Success Mathematical Self-Refinement via Trajectory-Aware Sampling in End-to-End Online RL](https://aclanthology.org/2026.emnlp-main.2544/) (Hao et al., EMNLP 2026)
ACL
- Yaqian Hao, Yifan Hao, Yankai Liu, Fangning Chao, Zhaojun Cui, Haiyu Zhang, Huan Bai, Fanyu Meng, Chao Deng, and Junlan Feng. 2026. FlipTS: Failure-to-Success Mathematical Self-Refinement via Trajectory-Aware Sampling in End-to-End Online RL. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 57845–57862, Budapest, Hungary. Association for Computational Linguistics.