@inproceedings{ghate-etal-2026-evaluesteer,
title = "{EVALUESTEER}: Measuring Reward Model Steerability Towards Values and Preferences",
author = "Ghate, Kshitish and
Liu, Andy and
Jain, Devansh and
Sorensen, Taylor and
Kasirzadeh, Atoosa and
Caliskan, Aylin and
Diab, Mona T. and
Sap, Maarten",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.572/",
pages = "12072--12099",
ISBN = "979-8-89176-462-0",
abstract = "As large language models (LLMs) are deployed globally, creating pluralistic systems that can accommodate the diverse preferences and values of users becomes essential. We introduce EVALUESTEER, a human-validated benchmark to measure the steerability of LLM and reward models (RMs) to users' value and stylistic preference profiles grounded in psychology and literature on human-LLM interaction. To address the gap in existing datasets that do not support controlled evaluations of RM steering, we synthetically generated 165,888 preference pairs {--} systematically varying pairs along 4 value dimensions (traditional, secular-rational, survival, and self-expression) and 4 style dimensions (verbosity, readability, confidence, warmth). We use EVALUESTEER to evaluate whether, given a user profile and a pair of candidate value and style laden responses, LLMs and RMs are able to select the output that aligns with the user{'}s preferences. Our results show that, when given the user{'}s full profile of values and stylistic preferences, the best models achieve $<$75{\%} accuracy in choosing the correct response, in contrast to $>$ 99{\%} accuracy when only relevant style and value preferences are provided. RMs also exhibit systematic biases, favoring secular values and verbose, confident language, and prioritizing style alignment over value alignment when the two conflict. EVALUESTEER thus highlights the limitations of current RMs at identifying and adapting to relevant user profile information, and provides a challenging testbed for developing RMs that can be steered towards diverse human values and preferences."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ghate-etal-2026-evaluesteer">
<titleInfo>
<title>EVALUESTEER: Measuring Reward Model Steerability Towards Values and Preferences</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kshitish</namePart>
<namePart type="family">Ghate</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andy</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Devansh</namePart>
<namePart type="family">Jain</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Taylor</namePart>
<namePart type="family">Sorensen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Atoosa</namePart>
<namePart type="family">Kasirzadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aylin</namePart>
<namePart type="family">Caliskan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mona</namePart>
<namePart type="given">T</namePart>
<namePart type="family">Diab</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maarten</namePart>
<namePart type="family">Sap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>As large language models (LLMs) are deployed globally, creating pluralistic systems that can accommodate the diverse preferences and values of users becomes essential. We introduce EVALUESTEER, a human-validated benchmark to measure the steerability of LLM and reward models (RMs) to users’ value and stylistic preference profiles grounded in psychology and literature on human-LLM interaction. To address the gap in existing datasets that do not support controlled evaluations of RM steering, we synthetically generated 165,888 preference pairs – systematically varying pairs along 4 value dimensions (traditional, secular-rational, survival, and self-expression) and 4 style dimensions (verbosity, readability, confidence, warmth). We use EVALUESTEER to evaluate whether, given a user profile and a pair of candidate value and style laden responses, LLMs and RMs are able to select the output that aligns with the user’s preferences. Our results show that, when given the user’s full profile of values and stylistic preferences, the best models achieve <75% accuracy in choosing the correct response, in contrast to > 99% accuracy when only relevant style and value preferences are provided. RMs also exhibit systematic biases, favoring secular values and verbose, confident language, and prioritizing style alignment over value alignment when the two conflict. EVALUESTEER thus highlights the limitations of current RMs at identifying and adapting to relevant user profile information, and provides a challenging testbed for developing RMs that can be steered towards diverse human values and preferences.</abstract>
<identifier type="citekey">ghate-etal-2026-evaluesteer</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.572/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>12072</start>
<end>12099</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T EVALUESTEER: Measuring Reward Model Steerability Towards Values and Preferences
%A Ghate, Kshitish
%A Liu, Andy
%A Jain, Devansh
%A Sorensen, Taylor
%A Kasirzadeh, Atoosa
%A Caliskan, Aylin
%A Diab, Mona T.
%A Sap, Maarten
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F ghate-etal-2026-evaluesteer
%X As large language models (LLMs) are deployed globally, creating pluralistic systems that can accommodate the diverse preferences and values of users becomes essential. We introduce EVALUESTEER, a human-validated benchmark to measure the steerability of LLM and reward models (RMs) to users’ value and stylistic preference profiles grounded in psychology and literature on human-LLM interaction. To address the gap in existing datasets that do not support controlled evaluations of RM steering, we synthetically generated 165,888 preference pairs – systematically varying pairs along 4 value dimensions (traditional, secular-rational, survival, and self-expression) and 4 style dimensions (verbosity, readability, confidence, warmth). We use EVALUESTEER to evaluate whether, given a user profile and a pair of candidate value and style laden responses, LLMs and RMs are able to select the output that aligns with the user’s preferences. Our results show that, when given the user’s full profile of values and stylistic preferences, the best models achieve <75% accuracy in choosing the correct response, in contrast to > 99% accuracy when only relevant style and value preferences are provided. RMs also exhibit systematic biases, favoring secular values and verbose, confident language, and prioritizing style alignment over value alignment when the two conflict. EVALUESTEER thus highlights the limitations of current RMs at identifying and adapting to relevant user profile information, and provides a challenging testbed for developing RMs that can be steered towards diverse human values and preferences.
%U https://aclanthology.org/2026.findings-emnlp.572/
%P 12072-12099
Markdown (Informal)
[EVALUESTEER: Measuring Reward Model Steerability Towards Values and Preferences](https://aclanthology.org/2026.findings-emnlp.572/) (Ghate et al., Findings 2026)
ACL
- Kshitish Ghate, Andy Liu, Devansh Jain, Taylor Sorensen, Atoosa Kasirzadeh, Aylin Caliskan, Mona T. Diab, and Maarten Sap. 2026. EVALUESTEER: Measuring Reward Model Steerability Towards Values and Preferences. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 12072–12099, Budapest, Hungary. Association for Computational Linguistics.