@inproceedings{liu-etal-2026-examining,
title = "Examining Reasoning {LLM}s-as-Judges in Non-Verifiable {LLM} Post-Training",
author = "Liu, Yixin and
Yu, Yue and
Su, DiJia and
Wang, Sid and
Wang, Xuewei and
Jiang, Song and
Liu, Bo and
Cohan, Arman and
Tian, Yuandong and
Chen, Zhengxing",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2496/",
pages = "56671--56704",
ISBN = "979-8-89176-459-0",
abstract = "Reasoning LLMs-as-Judges, which benefit from inference-time scaling, offer a promising path for extending the success of reasoning models to non-verifiable domains, where output quality cannot be directly checked. While reasoning judges have shown gains on static evaluation benchmarks, their impact on actual LLM post-training remains underexplored. Therefore, we conduct a controlled study of non-reasoning and reasoning LLM-judges in reinforcement-learning-based LLM alignment. In our synthetic setting, a ``gold'' judge provides preference annotations for training smaller judges and evaluates the resulting policies. We find that non-reasoning judges lead to the standard reward hacking pattern: policies receive higher rewards from the training judge but lower rewards from the gold judge. Reasoning judges, in contrast, produce policies with high gold-judge rewards. However, a qualitative analysis shows that these high rewards arise from adversarial outputs that exploit the gold judge{'}s preferences and transfer to other LLM-judges on benchmarks like Arena-Hard. Together with our ablations on judge training design choices, these findings reveal both the promise and risks of using reasoning LLM-judges for non-verifiable LLM post-training."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-examining">
<titleInfo>
<title>Examining Reasoning LLMs-as-Judges in Non-Verifiable LLM Post-Training</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yixin</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">DiJia</namePart>
<namePart type="family">Su</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sid</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xuewei</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Song</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bo</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Arman</namePart>
<namePart type="family">Cohan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuandong</namePart>
<namePart type="family">Tian</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhengxing</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Reasoning LLMs-as-Judges, which benefit from inference-time scaling, offer a promising path for extending the success of reasoning models to non-verifiable domains, where output quality cannot be directly checked. While reasoning judges have shown gains on static evaluation benchmarks, their impact on actual LLM post-training remains underexplored. Therefore, we conduct a controlled study of non-reasoning and reasoning LLM-judges in reinforcement-learning-based LLM alignment. In our synthetic setting, a “gold” judge provides preference annotations for training smaller judges and evaluates the resulting policies. We find that non-reasoning judges lead to the standard reward hacking pattern: policies receive higher rewards from the training judge but lower rewards from the gold judge. Reasoning judges, in contrast, produce policies with high gold-judge rewards. However, a qualitative analysis shows that these high rewards arise from adversarial outputs that exploit the gold judge’s preferences and transfer to other LLM-judges on benchmarks like Arena-Hard. Together with our ablations on judge training design choices, these findings reveal both the promise and risks of using reasoning LLM-judges for non-verifiable LLM post-training.</abstract>
<identifier type="citekey">liu-etal-2026-examining</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2496/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>56671</start>
<end>56704</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Examining Reasoning LLMs-as-Judges in Non-Verifiable LLM Post-Training
%A Liu, Yixin
%A Yu, Yue
%A Su, DiJia
%A Wang, Sid
%A Wang, Xuewei
%A Jiang, Song
%A Liu, Bo
%A Cohan, Arman
%A Tian, Yuandong
%A Chen, Zhengxing
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F liu-etal-2026-examining
%X Reasoning LLMs-as-Judges, which benefit from inference-time scaling, offer a promising path for extending the success of reasoning models to non-verifiable domains, where output quality cannot be directly checked. While reasoning judges have shown gains on static evaluation benchmarks, their impact on actual LLM post-training remains underexplored. Therefore, we conduct a controlled study of non-reasoning and reasoning LLM-judges in reinforcement-learning-based LLM alignment. In our synthetic setting, a “gold” judge provides preference annotations for training smaller judges and evaluates the resulting policies. We find that non-reasoning judges lead to the standard reward hacking pattern: policies receive higher rewards from the training judge but lower rewards from the gold judge. Reasoning judges, in contrast, produce policies with high gold-judge rewards. However, a qualitative analysis shows that these high rewards arise from adversarial outputs that exploit the gold judge’s preferences and transfer to other LLM-judges on benchmarks like Arena-Hard. Together with our ablations on judge training design choices, these findings reveal both the promise and risks of using reasoning LLM-judges for non-verifiable LLM post-training.
%U https://aclanthology.org/2026.emnlp-main.2496/
%P 56671-56704
Markdown (Informal)
[Examining Reasoning LLMs-as-Judges in Non-Verifiable LLM Post-Training](https://aclanthology.org/2026.emnlp-main.2496/) (Liu et al., EMNLP 2026)
ACL
- Yixin Liu, Yue Yu, DiJia Su, Sid Wang, Xuewei Wang, Song Jiang, Bo Liu, Arman Cohan, Yuandong Tian, and Zhengxing Chen. 2026. Examining Reasoning LLMs-as-Judges in Non-Verifiable LLM Post-Training. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 56671–56704, Budapest, Hungary. Association for Computational Linguistics.