@inproceedings{roy-etal-2026-premise,
title = "{PR}e{MISE}: Policy Rubrics as Measurement Specifications for {LLM} Judges",
author = "Roy, Swastik and
Pujari, Rajkumar and
Kumarage, Tharindu and
Peris, Charith and
Gupta, Rahul and
Rumshisky, Anna and
Natarajan, Pradeep and
Saligrama, Venkatesh",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2653/",
pages = "60382--60406",
ISBN = "979-8-89176-459-0",
abstract = "LLM judges are increasingly used to evaluate open-ended responses, but their scores depend strongly on the rubrics that condition them. A vague rubric asking for a response to be ``helpful and factual'' can reward polished answers that invent facts or violate user intent. We treat reusable rubrics as measurement specifications: changing the rubric changes the response quality measurement induced by a fixed judge. We introduce \textsc{PReMISE}, a framework that, given pairwise human-preference data, (i) discovers a policy-level rubric set, and (ii) audits any rubric set under LLM-judge use along four axes: structural adequacy, reliability, preference fit, and adversarial robustness. We evaluate \textsc{PReMISE} on four public human-preference corpora {--} UltraFeedback, Arena-Expert-5K, HelpSteer3, and HH-RLHF {--} across six rubric sources and three open-weight judges spanning 32B to 1T parameters. Across rubric sources no raw source is simultaneously reliable, preference-predictive, and adversarially robust; and high inter-rater agreement does not imply low exploitability. \textsc{PReMISE} is the only rubric source to score non-trivially on applicability, specificity, and effective dimensionality simultaneously. We contribute two audit-targeted repair operations: preference-rank selection raises held-out cross-source, cross-judge mean accuracy on paired responses from 65.0{\%} to 68.6{\%}, competitive with the strongest rubric-discovery baselines and leading on two of three judges in our cross-judge sweep; reliability-constrained refinement reduces the rate at which exploit responses receive high scores from 46.4{\%} to 36.0{\%} with little change in inter-judge agreement ($\alpha{=}.531\to.519$)."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="roy-etal-2026-premise">
<titleInfo>
<title>PReMISE: Policy Rubrics as Measurement Specifications for LLM Judges</title>
</titleInfo>
<name type="personal">
<namePart type="given">Swastik</namePart>
<namePart type="family">Roy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rajkumar</namePart>
<namePart type="family">Pujari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tharindu</namePart>
<namePart type="family">Kumarage</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Charith</namePart>
<namePart type="family">Peris</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rahul</namePart>
<namePart type="family">Gupta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="family">Rumshisky</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pradeep</namePart>
<namePart type="family">Natarajan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Venkatesh</namePart>
<namePart type="family">Saligrama</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>LLM judges are increasingly used to evaluate open-ended responses, but their scores depend strongly on the rubrics that condition them. A vague rubric asking for a response to be “helpful and factual” can reward polished answers that invent facts or violate user intent. We treat reusable rubrics as measurement specifications: changing the rubric changes the response quality measurement induced by a fixed judge. We introduce PReMISE, a framework that, given pairwise human-preference data, (i) discovers a policy-level rubric set, and (ii) audits any rubric set under LLM-judge use along four axes: structural adequacy, reliability, preference fit, and adversarial robustness. We evaluate PReMISE on four public human-preference corpora – UltraFeedback, Arena-Expert-5K, HelpSteer3, and HH-RLHF – across six rubric sources and three open-weight judges spanning 32B to 1T parameters. Across rubric sources no raw source is simultaneously reliable, preference-predictive, and adversarially robust; and high inter-rater agreement does not imply low exploitability. PReMISE is the only rubric source to score non-trivially on applicability, specificity, and effective dimensionality simultaneously. We contribute two audit-targeted repair operations: preference-rank selection raises held-out cross-source, cross-judge mean accuracy on paired responses from 65.0% to 68.6%, competitive with the strongest rubric-discovery baselines and leading on two of three judges in our cross-judge sweep; reliability-constrained refinement reduces the rate at which exploit responses receive high scores from 46.4% to 36.0% with little change in inter-judge agreement (α=.531.519).</abstract>
<identifier type="citekey">roy-etal-2026-premise</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2653/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>60382</start>
<end>60406</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T PReMISE: Policy Rubrics as Measurement Specifications for LLM Judges
%A Roy, Swastik
%A Pujari, Rajkumar
%A Kumarage, Tharindu
%A Peris, Charith
%A Gupta, Rahul
%A Rumshisky, Anna
%A Natarajan, Pradeep
%A Saligrama, Venkatesh
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F roy-etal-2026-premise
%X LLM judges are increasingly used to evaluate open-ended responses, but their scores depend strongly on the rubrics that condition them. A vague rubric asking for a response to be “helpful and factual” can reward polished answers that invent facts or violate user intent. We treat reusable rubrics as measurement specifications: changing the rubric changes the response quality measurement induced by a fixed judge. We introduce PReMISE, a framework that, given pairwise human-preference data, (i) discovers a policy-level rubric set, and (ii) audits any rubric set under LLM-judge use along four axes: structural adequacy, reliability, preference fit, and adversarial robustness. We evaluate PReMISE on four public human-preference corpora – UltraFeedback, Arena-Expert-5K, HelpSteer3, and HH-RLHF – across six rubric sources and three open-weight judges spanning 32B to 1T parameters. Across rubric sources no raw source is simultaneously reliable, preference-predictive, and adversarially robust; and high inter-rater agreement does not imply low exploitability. PReMISE is the only rubric source to score non-trivially on applicability, specificity, and effective dimensionality simultaneously. We contribute two audit-targeted repair operations: preference-rank selection raises held-out cross-source, cross-judge mean accuracy on paired responses from 65.0% to 68.6%, competitive with the strongest rubric-discovery baselines and leading on two of three judges in our cross-judge sweep; reliability-constrained refinement reduces the rate at which exploit responses receive high scores from 46.4% to 36.0% with little change in inter-judge agreement (α=.531.519).
%U https://aclanthology.org/2026.emnlp-main.2653/
%P 60382-60406
Markdown (Informal)
[PReMISE: Policy Rubrics as Measurement Specifications for LLM Judges](https://aclanthology.org/2026.emnlp-main.2653/) (Roy et al., EMNLP 2026)
ACL
- Swastik Roy, Rajkumar Pujari, Tharindu Kumarage, Charith Peris, Rahul Gupta, Anna Rumshisky, Pradeep Natarajan, and Venkatesh Saligrama. 2026. PReMISE: Policy Rubrics as Measurement Specifications for LLM Judges. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 60382–60406, Budapest, Hungary. Association for Computational Linguistics.