@inproceedings{kumar-2026-decomposing,
title = "Decomposing {LLM}-Judge{--}Human Agreement: How Much Is Inherited Surface Preference?",
author = "Kumar, Gaurav",
editor = "Sedoc, Jo{\~a}o and
Shu, Raphael and
Drozd, Aleksandr and
Akula, Arjun and
Rogers, Anna and
Giorgi, Salvatore and
Benton, Adrian",
booktitle = "Proceedings of the Seventh Workshop on Insights from Negative Results in {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.insights-1.7/",
pages = "60--69",
ISBN = "979-8-89176-480-4",
abstract = "LLM-as-judge evaluation is typically validated using judge-human agreement with standard benchmarks reporting agreement rates of roughly 80-85 {\%}. We show that this aggregate metric can be misleading. Most of that agreement comes from pairs where both judge and human prefer the longer, better-formatted response, rather than from agreement on response quality. On the remaining pairs, agreement drops to chance levels on MT-Bench, and below chance when surface cues mislead. We show this with two simple logistic-regression baselines on length, structure, and lexical features: one predicts the human{'}s pick, one predicts the judge{'}s. The pattern holds across closed and open judges and both benchmarks, and is strongest on the expert annotations the field treats as the quality gold standard. We suggest reporting the inheritance share alongside aggregate agreement: a simple reporting metric we defined that shows whether a judge agrees with humans for the right reasons."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kumar-2026-decomposing">
<titleInfo>
<title>Decomposing LLM-Judge–Human Agreement: How Much Is Inherited Surface Preference?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Gaurav</namePart>
<namePart type="family">Kumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Seventh Workshop on Insights from Negative Results in NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">João</namePart>
<namePart type="family">Sedoc</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raphael</namePart>
<namePart type="family">Shu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aleksandr</namePart>
<namePart type="family">Drozd</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Arjun</namePart>
<namePart type="family">Akula</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="family">Rogers</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Salvatore</namePart>
<namePart type="family">Giorgi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Adrian</namePart>
<namePart type="family">Benton</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-480-4</identifier>
</relatedItem>
<abstract>LLM-as-judge evaluation is typically validated using judge-human agreement with standard benchmarks reporting agreement rates of roughly 80-85 %. We show that this aggregate metric can be misleading. Most of that agreement comes from pairs where both judge and human prefer the longer, better-formatted response, rather than from agreement on response quality. On the remaining pairs, agreement drops to chance levels on MT-Bench, and below chance when surface cues mislead. We show this with two simple logistic-regression baselines on length, structure, and lexical features: one predicts the human’s pick, one predicts the judge’s. The pattern holds across closed and open judges and both benchmarks, and is strongest on the expert annotations the field treats as the quality gold standard. We suggest reporting the inheritance share alongside aggregate agreement: a simple reporting metric we defined that shows whether a judge agrees with humans for the right reasons.</abstract>
<identifier type="citekey">kumar-2026-decomposing</identifier>
<location>
<url>https://aclanthology.org/2026.insights-1.7/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>60</start>
<end>69</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Decomposing LLM-Judge–Human Agreement: How Much Is Inherited Surface Preference?
%A Kumar, Gaurav
%Y Sedoc, João
%Y Shu, Raphael
%Y Drozd, Aleksandr
%Y Akula, Arjun
%Y Rogers, Anna
%Y Giorgi, Salvatore
%Y Benton, Adrian
%S Proceedings of the Seventh Workshop on Insights from Negative Results in NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-480-4
%F kumar-2026-decomposing
%X LLM-as-judge evaluation is typically validated using judge-human agreement with standard benchmarks reporting agreement rates of roughly 80-85 %. We show that this aggregate metric can be misleading. Most of that agreement comes from pairs where both judge and human prefer the longer, better-formatted response, rather than from agreement on response quality. On the remaining pairs, agreement drops to chance levels on MT-Bench, and below chance when surface cues mislead. We show this with two simple logistic-regression baselines on length, structure, and lexical features: one predicts the human’s pick, one predicts the judge’s. The pattern holds across closed and open judges and both benchmarks, and is strongest on the expert annotations the field treats as the quality gold standard. We suggest reporting the inheritance share alongside aggregate agreement: a simple reporting metric we defined that shows whether a judge agrees with humans for the right reasons.
%U https://aclanthology.org/2026.insights-1.7/
%P 60-69
Markdown (Informal)
[Decomposing LLM-Judge–Human Agreement: How Much Is Inherited Surface Preference?](https://aclanthology.org/2026.insights-1.7/) (Kumar, insights 2026)
ACL