@inproceedings{li-etal-2026-reward,
title = "Reward Yourself: Efficient Self Rewards for Trustworthy Sampling",
author = "Li, Mingjie and
Si, Wai Man and
Backes, Michael and
Zhang, Yang",
editor = "Liakata, Maria and
Moreira, Viviane P. and
Zhang, Jiajun and
Jurgens, David",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2026",
month = jul,
year = "2026",
address = "San Diego, California, United States",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-acl.1554/",
pages = "31070--31086",
ISBN = "979-8-89176-395-1",
abstract = "As high-quality data becomes harder to obtain, reward models are increasingly important. Beyond the costly RLHF stage, they are now used at inference time to guide LLM generation and in data selection for post-training. These methods bring efficiency and performance gains, but current reward models often fail to prevent untrustworthy behaviors such as privacy leaks and stereotypes. Re-training reward models to address these issues is expensive, since it requires large-scale human preference data. We propose SelfRW, a lightweight intrinsic reward that needs no extra fine-tuning or auxiliary models. By pruning current LLMs to approximate an ``trust'' and an ``untrust'' token distribution, we compute the log-probability difference as an auxiliary reward. When integrated into reward-guided sampling, SelfRW significantly reduces untrustworthy outputs while preserving task performance. It also improves reward-guided data selection, yielding better post-trained models. Experiments with two reward models and four LLMs on privacy, bias, and stereotype benchmarks show that combining SelfRW consistently improves trustworthiness (over 10{\%} in privacy tasks and 20{\%} in bias tasks) with minimal impact on general utility benchmarks."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-etal-2026-reward">
<titleInfo>
<title>Reward Yourself: Efficient Self Rewards for Trustworthy Sampling</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mingjie</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wai</namePart>
<namePart type="given">Man</namePart>
<namePart type="family">Si</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Michael</namePart>
<namePart type="family">Backes</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yang</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: ACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Maria</namePart>
<namePart type="family">Liakata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Viviane</namePart>
<namePart type="given">P</namePart>
<namePart type="family">Moreira</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiajun</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Jurgens</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">San Diego, California, United States</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-395-1</identifier>
</relatedItem>
<abstract>As high-quality data becomes harder to obtain, reward models are increasingly important. Beyond the costly RLHF stage, they are now used at inference time to guide LLM generation and in data selection for post-training. These methods bring efficiency and performance gains, but current reward models often fail to prevent untrustworthy behaviors such as privacy leaks and stereotypes. Re-training reward models to address these issues is expensive, since it requires large-scale human preference data. We propose SelfRW, a lightweight intrinsic reward that needs no extra fine-tuning or auxiliary models. By pruning current LLMs to approximate an “trust” and an “untrust” token distribution, we compute the log-probability difference as an auxiliary reward. When integrated into reward-guided sampling, SelfRW significantly reduces untrustworthy outputs while preserving task performance. It also improves reward-guided data selection, yielding better post-trained models. Experiments with two reward models and four LLMs on privacy, bias, and stereotype benchmarks show that combining SelfRW consistently improves trustworthiness (over 10% in privacy tasks and 20% in bias tasks) with minimal impact on general utility benchmarks.</abstract>
<identifier type="citekey">li-etal-2026-reward</identifier>
<location>
<url>https://aclanthology.org/2026.findings-acl.1554/</url>
</location>
<part>
<date>2026-07</date>
<extent unit="page">
<start>31070</start>
<end>31086</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Reward Yourself: Efficient Self Rewards for Trustworthy Sampling
%A Li, Mingjie
%A Si, Wai Man
%A Backes, Michael
%A Zhang, Yang
%Y Liakata, Maria
%Y Moreira, Viviane P.
%Y Zhang, Jiajun
%Y Jurgens, David
%S Findings of the Association for Computational Linguistics: ACL 2026
%D 2026
%8 July
%I Association for Computational Linguistics
%C San Diego, California, United States
%@ 979-8-89176-395-1
%F li-etal-2026-reward
%X As high-quality data becomes harder to obtain, reward models are increasingly important. Beyond the costly RLHF stage, they are now used at inference time to guide LLM generation and in data selection for post-training. These methods bring efficiency and performance gains, but current reward models often fail to prevent untrustworthy behaviors such as privacy leaks and stereotypes. Re-training reward models to address these issues is expensive, since it requires large-scale human preference data. We propose SelfRW, a lightweight intrinsic reward that needs no extra fine-tuning or auxiliary models. By pruning current LLMs to approximate an “trust” and an “untrust” token distribution, we compute the log-probability difference as an auxiliary reward. When integrated into reward-guided sampling, SelfRW significantly reduces untrustworthy outputs while preserving task performance. It also improves reward-guided data selection, yielding better post-trained models. Experiments with two reward models and four LLMs on privacy, bias, and stereotype benchmarks show that combining SelfRW consistently improves trustworthiness (over 10% in privacy tasks and 20% in bias tasks) with minimal impact on general utility benchmarks.
%U https://aclanthology.org/2026.findings-acl.1554/
%P 31070-31086
Markdown (Informal)
[Reward Yourself: Efficient Self Rewards for Trustworthy Sampling](https://aclanthology.org/2026.findings-acl.1554/) (Li et al., Findings 2026)
ACL