@inproceedings{loftus-etal-2026-risk,
title = "Risk-Averse Failure Modes of {J}ensen-{S}hannon Distance in {RLVR} Distributional Alignment",
author = "Loftus, Sebastian and
Wang, Xinpeng and
De Marneffe, Marie-Catherine and
Plank, Barbara",
editor = "Sedoc, Jo{\~a}o and
Shu, Raphael and
Drozd, Aleksandr and
Akula, Arjun and
Rogers, Anna and
Giorgi, Salvatore and
Benton, Adrian",
booktitle = "Proceedings of the Seventh Workshop on Insights from Negative Results in {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.insights-1.4/",
pages = "33--42",
ISBN = "979-8-89176-480-4",
abstract = "Jensen-Shannon Distance (JSD) appears well-suited as a reward signal for aligning model predictions with human annotations, particularly for subjective tasks where label variation reflects legitimate disagreement. We demonstrate that JSD{'}s mathematical properties create exploitable reward landscapes in reinforcement learning. Via a study on preference distribution prediction, we show that two JSD-based reward formulations both fail to achieve alignment and instead lead to models converging at conservative, near-uniform predictions that maximize expected reward without matching human behavior. We trace this failure to JSD{'}s non-uniform sensitivity: it penalizes errors near balanced distributions less than errors near extreme distributions, creating risk asymmetry that incentivizes safe predictions over accurate ones. Our findings reveal fundamental challenges in using bounded symmetric divergence metrics as scalar rewards for distributional alignment."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="loftus-etal-2026-risk">
<titleInfo>
<title>Risk-Averse Failure Modes of Jensen-Shannon Distance in RLVR Distributional Alignment</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sebastian</namePart>
<namePart type="family">Loftus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinpeng</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marie-Catherine</namePart>
<namePart type="family">De Marneffe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Seventh Workshop on Insights from Negative Results in NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">João</namePart>
<namePart type="family">Sedoc</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raphael</namePart>
<namePart type="family">Shu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aleksandr</namePart>
<namePart type="family">Drozd</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Arjun</namePart>
<namePart type="family">Akula</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="family">Rogers</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Salvatore</namePart>
<namePart type="family">Giorgi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Adrian</namePart>
<namePart type="family">Benton</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-480-4</identifier>
</relatedItem>
<abstract>Jensen-Shannon Distance (JSD) appears well-suited as a reward signal for aligning model predictions with human annotations, particularly for subjective tasks where label variation reflects legitimate disagreement. We demonstrate that JSD’s mathematical properties create exploitable reward landscapes in reinforcement learning. Via a study on preference distribution prediction, we show that two JSD-based reward formulations both fail to achieve alignment and instead lead to models converging at conservative, near-uniform predictions that maximize expected reward without matching human behavior. We trace this failure to JSD’s non-uniform sensitivity: it penalizes errors near balanced distributions less than errors near extreme distributions, creating risk asymmetry that incentivizes safe predictions over accurate ones. Our findings reveal fundamental challenges in using bounded symmetric divergence metrics as scalar rewards for distributional alignment.</abstract>
<identifier type="citekey">loftus-etal-2026-risk</identifier>
<location>
<url>https://aclanthology.org/2026.insights-1.4/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>33</start>
<end>42</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Risk-Averse Failure Modes of Jensen-Shannon Distance in RLVR Distributional Alignment
%A Loftus, Sebastian
%A Wang, Xinpeng
%A De Marneffe, Marie-Catherine
%A Plank, Barbara
%Y Sedoc, João
%Y Shu, Raphael
%Y Drozd, Aleksandr
%Y Akula, Arjun
%Y Rogers, Anna
%Y Giorgi, Salvatore
%Y Benton, Adrian
%S Proceedings of the Seventh Workshop on Insights from Negative Results in NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-480-4
%F loftus-etal-2026-risk
%X Jensen-Shannon Distance (JSD) appears well-suited as a reward signal for aligning model predictions with human annotations, particularly for subjective tasks where label variation reflects legitimate disagreement. We demonstrate that JSD’s mathematical properties create exploitable reward landscapes in reinforcement learning. Via a study on preference distribution prediction, we show that two JSD-based reward formulations both fail to achieve alignment and instead lead to models converging at conservative, near-uniform predictions that maximize expected reward without matching human behavior. We trace this failure to JSD’s non-uniform sensitivity: it penalizes errors near balanced distributions less than errors near extreme distributions, creating risk asymmetry that incentivizes safe predictions over accurate ones. Our findings reveal fundamental challenges in using bounded symmetric divergence metrics as scalar rewards for distributional alignment.
%U https://aclanthology.org/2026.insights-1.4/
%P 33-42
Markdown (Informal)
[Risk-Averse Failure Modes of Jensen-Shannon Distance in RLVR Distributional Alignment](https://aclanthology.org/2026.insights-1.4/) (Loftus et al., insights 2026)
ACL