@inproceedings{ngo-etal-2026-verifiable,
title = "Not All Verifiable Preference Pairs Are Equal: Support-Aware Filtering for Stable Iterative {DPO}",
author = "Ngo, Viet Anh and
Son, Long Tri Thai and
Duc, Nguyen Viet and
Duc, Dong Minh and
Tran, Quang Minh and
Linh, Nguyen Thi Thuy and
Ha, Nguyen Viet",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 2: Short Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-short.51/",
pages = "807--815",
ISBN = "979-8-89176-440-8",
abstract = "Direct Preference Optimization (DPO) is well suited to verifiable reasoning tasks because preference pairs can be constructed automatically from answer correctness. However, in iterative DPO, correctness alone does not determine whether a pair provides a useful training signal. We distinguish two regimes according to the origin of the chosen response: On-policy pairs, where both the chosen and rejected responses are sampled from the current policy, and Fallback pairs, where the policy fails to generate a correct response and the chosen completion is supplied by a dataset-provided reference solution. Based on this distinction, we propose support-aware filtering, which applies margin-based filtering to On-policy pairs and a likelihood-based support gate to Fallback pairs. Experiments on GSM8K and MATH-500 with Qwen2.5-style 0.5B, 1.5B, and 3B models show that support-aware filtering yields its clearest gains on MATH-500 at 1.5B and 3B, while GSM8K and the 0.5B MATH-500 setting are more mixed. Overall, our findings suggest that preference margins in iterative verifiable DPO are pair-origin dependent, and that chosen-response support is a useful signal for stabilizing fallback supervision."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ngo-etal-2026-verifiable">
<titleInfo>
<title>Not All Verifiable Preference Pairs Are Equal: Support-Aware Filtering for Stable Iterative DPO</title>
</titleInfo>
<name type="personal">
<namePart type="given">Viet</namePart>
<namePart type="given">Anh</namePart>
<namePart type="family">Ngo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Long</namePart>
<namePart type="given">Tri</namePart>
<namePart type="given">Thai</namePart>
<namePart type="family">Son</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nguyen</namePart>
<namePart type="given">Viet</namePart>
<namePart type="family">Duc</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dong</namePart>
<namePart type="given">Minh</namePart>
<namePart type="family">Duc</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Quang</namePart>
<namePart type="given">Minh</namePart>
<namePart type="family">Tran</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nguyen</namePart>
<namePart type="given">Thi</namePart>
<namePart type="given">Thuy</namePart>
<namePart type="family">Linh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nguyen</namePart>
<namePart type="given">Viet</namePart>
<namePart type="family">Ha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-440-8</identifier>
</relatedItem>
<abstract>Direct Preference Optimization (DPO) is well suited to verifiable reasoning tasks because preference pairs can be constructed automatically from answer correctness. However, in iterative DPO, correctness alone does not determine whether a pair provides a useful training signal. We distinguish two regimes according to the origin of the chosen response: On-policy pairs, where both the chosen and rejected responses are sampled from the current policy, and Fallback pairs, where the policy fails to generate a correct response and the chosen completion is supplied by a dataset-provided reference solution. Based on this distinction, we propose support-aware filtering, which applies margin-based filtering to On-policy pairs and a likelihood-based support gate to Fallback pairs. Experiments on GSM8K and MATH-500 with Qwen2.5-style 0.5B, 1.5B, and 3B models show that support-aware filtering yields its clearest gains on MATH-500 at 1.5B and 3B, while GSM8K and the 0.5B MATH-500 setting are more mixed. Overall, our findings suggest that preference margins in iterative verifiable DPO are pair-origin dependent, and that chosen-response support is a useful signal for stabilizing fallback supervision.</abstract>
<identifier type="citekey">ngo-etal-2026-verifiable</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-short.51/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>807</start>
<end>815</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Not All Verifiable Preference Pairs Are Equal: Support-Aware Filtering for Stable Iterative DPO
%A Ngo, Viet Anh
%A Son, Long Tri Thai
%A Duc, Nguyen Viet
%A Duc, Dong Minh
%A Tran, Quang Minh
%A Linh, Nguyen Thi Thuy
%A Ha, Nguyen Viet
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-440-8
%F ngo-etal-2026-verifiable
%X Direct Preference Optimization (DPO) is well suited to verifiable reasoning tasks because preference pairs can be constructed automatically from answer correctness. However, in iterative DPO, correctness alone does not determine whether a pair provides a useful training signal. We distinguish two regimes according to the origin of the chosen response: On-policy pairs, where both the chosen and rejected responses are sampled from the current policy, and Fallback pairs, where the policy fails to generate a correct response and the chosen completion is supplied by a dataset-provided reference solution. Based on this distinction, we propose support-aware filtering, which applies margin-based filtering to On-policy pairs and a likelihood-based support gate to Fallback pairs. Experiments on GSM8K and MATH-500 with Qwen2.5-style 0.5B, 1.5B, and 3B models show that support-aware filtering yields its clearest gains on MATH-500 at 1.5B and 3B, while GSM8K and the 0.5B MATH-500 setting are more mixed. Overall, our findings suggest that preference margins in iterative verifiable DPO are pair-origin dependent, and that chosen-response support is a useful signal for stabilizing fallback supervision.
%U https://aclanthology.org/2026.aacl-short.51/
%P 807-815
Markdown (Informal)
[Not All Verifiable Preference Pairs Are Equal: Support-Aware Filtering for Stable Iterative DPO](https://aclanthology.org/2026.aacl-short.51/) (Ngo et al., AACL-IJCNLP 2026)
ACL
- Viet Anh Ngo, Long Tri Thai Son, Nguyen Viet Duc, Dong Minh Duc, Quang Minh Tran, Nguyen Thi Thuy Linh, and Nguyen Viet Ha. 2026. Not All Verifiable Preference Pairs Are Equal: Support-Aware Filtering for Stable Iterative DPO. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers), pages 807–815, Hengqin, China. Association for Computational Linguistics.