@inproceedings{cao-etal-2026-scar,
title = "{SCAR}: Shapley Credit Assignment for More Efficient {RLHF}",
author = "Cao, Meng and
Zhang, Shuyuan and
Chang, Xiao-Wen and
Precup, Doina",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1795/",
pages = "40898--40915",
ISBN = "979-8-89176-459-0",
abstract = "Reinforcement Learning from Human Feedback (RLHF) often suffers from sparse reward signals, making credit assignment to tokens challenging. We propose Shapley Credit Assignment Rewards (SCAR), a method that uses Shapley values from cooperative game theory to distribute sequence-level rewards across tokens or text spans according to their marginal contributions. SCAR provides dense rewards without requiring auxiliary critique models or fine-grained human annotations, while offering a principled game-theoretic foundation for credit assignment. We show theoretically that SCAR preserves the original optimal policy, and empirically demonstrate across sentiment control, summarization, and instruction tuning tasks that it converges faster and achieves higher final rewards than standard RLHF and attention-based dense reward baselines. To improve scalability on long sequences, we further introduce an Owen value{--}inspired adaptive text segmentation strategy. Our findings suggest that SCAR provides a more effective and theoretically sound method for credit assignment in RLHF, leading to more efficient alignment of LLMs."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="cao-etal-2026-scar">
<titleInfo>
<title>SCAR: Shapley Credit Assignment for More Efficient RLHF</title>
</titleInfo>
<name type="personal">
<namePart type="given">Meng</namePart>
<namePart type="family">Cao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shuyuan</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiao-Wen</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Doina</namePart>
<namePart type="family">Precup</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Reinforcement Learning from Human Feedback (RLHF) often suffers from sparse reward signals, making credit assignment to tokens challenging. We propose Shapley Credit Assignment Rewards (SCAR), a method that uses Shapley values from cooperative game theory to distribute sequence-level rewards across tokens or text spans according to their marginal contributions. SCAR provides dense rewards without requiring auxiliary critique models or fine-grained human annotations, while offering a principled game-theoretic foundation for credit assignment. We show theoretically that SCAR preserves the original optimal policy, and empirically demonstrate across sentiment control, summarization, and instruction tuning tasks that it converges faster and achieves higher final rewards than standard RLHF and attention-based dense reward baselines. To improve scalability on long sequences, we further introduce an Owen value–inspired adaptive text segmentation strategy. Our findings suggest that SCAR provides a more effective and theoretically sound method for credit assignment in RLHF, leading to more efficient alignment of LLMs.</abstract>
<identifier type="citekey">cao-etal-2026-scar</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1795/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>40898</start>
<end>40915</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SCAR: Shapley Credit Assignment for More Efficient RLHF
%A Cao, Meng
%A Zhang, Shuyuan
%A Chang, Xiao-Wen
%A Precup, Doina
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F cao-etal-2026-scar
%X Reinforcement Learning from Human Feedback (RLHF) often suffers from sparse reward signals, making credit assignment to tokens challenging. We propose Shapley Credit Assignment Rewards (SCAR), a method that uses Shapley values from cooperative game theory to distribute sequence-level rewards across tokens or text spans according to their marginal contributions. SCAR provides dense rewards without requiring auxiliary critique models or fine-grained human annotations, while offering a principled game-theoretic foundation for credit assignment. We show theoretically that SCAR preserves the original optimal policy, and empirically demonstrate across sentiment control, summarization, and instruction tuning tasks that it converges faster and achieves higher final rewards than standard RLHF and attention-based dense reward baselines. To improve scalability on long sequences, we further introduce an Owen value–inspired adaptive text segmentation strategy. Our findings suggest that SCAR provides a more effective and theoretically sound method for credit assignment in RLHF, leading to more efficient alignment of LLMs.
%U https://aclanthology.org/2026.emnlp-main.1795/
%P 40898-40915
Markdown (Informal)
[SCAR: Shapley Credit Assignment for More Efficient RLHF](https://aclanthology.org/2026.emnlp-main.1795/) (Cao et al., EMNLP 2026)
ACL
- Meng Cao, Shuyuan Zhang, Xiao-Wen Chang, and Doina Precup. 2026. SCAR: Shapley Credit Assignment for More Efficient RLHF. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 40898–40915, Budapest, Hungary. Association for Computational Linguistics.