@inproceedings{akarajaradwong-etal-2026-beyond,
title = "Beyond the Single-Expert Ceiling: Localizing {LLM}-as-Judge Stability on Free-Form Legal Essays",
author = "Akarajaradwong, Pawitsapak and
Lertprasertphakorn, Wuttikrai and
Chaksangchaichot, Chompakorn and
Nutanong, Sarana",
editor = "Aletras, Nikolaos and
Chalkidis, Ilias and
Barrett, Leslie and
Goanț{\u{a}}, C{\u{a}}t{\u{a}}lina and
Preoțiuc-Pietro, Daniel and
Spanakis, Gerasimos and
Xu, Shanshan",
booktitle = "Proceedings of the Natural Legal Language Processing Workshop 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.nllp-1.25/",
pages = "290--302",
ISBN = "979-8-89176-494-1",
abstract = "Free-form legal essay evaluation in NLP treats expert inter-rater stability as a single ceiling number: a correlation score between credentialed examiners is reported and used as the target that an LLM judge should approach. Whether that ceiling is uniform across items has rarely been tested on the expert side. We test it on Thai bar exam essays. Three Bar Council examiners and three anchor LLM judges (Claude 4.6 Opus, Gemini 3.1 Pro, GPT-5.4 Pro) independently graded the same 15 answers from the same four inputs: question, official rubric, gold answer, and student answer. On 10 of the 15 answers, both panels reach $\alpha = 0.75$. On the remaining 5, drawn from one question where the rubric does not prescribe how to grade a correct answer that omits a decisive citation, the human panel collapses to $\alpha = -0.34$ while the LLM panel holds at $\alpha = 0.68$. One examiner{'}s stricter reading of this underspecified rubric drives the gap. The cluster pattern replicates across seven of eight LLMs that took the exam in Phase{~}1 and were reused as judges, regardless of their own exam scores: the same model behaves differently in the two roles. The expert stability ceiling is not flat: free-form essay rubrics combine a deterministic correctness element (grounded by the gold answer) with an interpretive quality element (judged without one), and expert disagreement lives entirely on the interpretive element. The bottleneck of free-form essay evaluation is the rubric{'}s interpretive coverage, not LLM capability."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="akarajaradwong-etal-2026-beyond">
<titleInfo>
<title>Beyond the Single-Expert Ceiling: Localizing LLM-as-Judge Stability on Free-Form Legal Essays</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pawitsapak</namePart>
<namePart type="family">Akarajaradwong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wuttikrai</namePart>
<namePart type="family">Lertprasertphakorn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chompakorn</namePart>
<namePart type="family">Chaksangchaichot</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sarana</namePart>
<namePart type="family">Nutanong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Natural Legal Language Processing Workshop 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nikolaos</namePart>
<namePart type="family">Aletras</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ilias</namePart>
<namePart type="family">Chalkidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leslie</namePart>
<namePart type="family">Barrett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cătălina</namePart>
<namePart type="family">Goanță</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Preoțiuc-Pietro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gerasimos</namePart>
<namePart type="family">Spanakis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shanshan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-494-1</identifier>
</relatedItem>
<abstract>Free-form legal essay evaluation in NLP treats expert inter-rater stability as a single ceiling number: a correlation score between credentialed examiners is reported and used as the target that an LLM judge should approach. Whether that ceiling is uniform across items has rarely been tested on the expert side. We test it on Thai bar exam essays. Three Bar Council examiners and three anchor LLM judges (Claude 4.6 Opus, Gemini 3.1 Pro, GPT-5.4 Pro) independently graded the same 15 answers from the same four inputs: question, official rubric, gold answer, and student answer. On 10 of the 15 answers, both panels reach α = 0.75. On the remaining 5, drawn from one question where the rubric does not prescribe how to grade a correct answer that omits a decisive citation, the human panel collapses to α = -0.34 while the LLM panel holds at α = 0.68. One examiner’s stricter reading of this underspecified rubric drives the gap. The cluster pattern replicates across seven of eight LLMs that took the exam in Phase 1 and were reused as judges, regardless of their own exam scores: the same model behaves differently in the two roles. The expert stability ceiling is not flat: free-form essay rubrics combine a deterministic correctness element (grounded by the gold answer) with an interpretive quality element (judged without one), and expert disagreement lives entirely on the interpretive element. The bottleneck of free-form essay evaluation is the rubric’s interpretive coverage, not LLM capability.</abstract>
<identifier type="citekey">akarajaradwong-etal-2026-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.nllp-1.25/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>290</start>
<end>302</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond the Single-Expert Ceiling: Localizing LLM-as-Judge Stability on Free-Form Legal Essays
%A Akarajaradwong, Pawitsapak
%A Lertprasertphakorn, Wuttikrai
%A Chaksangchaichot, Chompakorn
%A Nutanong, Sarana
%Y Aletras, Nikolaos
%Y Chalkidis, Ilias
%Y Barrett, Leslie
%Y Goanță, Cătălina
%Y Preoțiuc-Pietro, Daniel
%Y Spanakis, Gerasimos
%Y Xu, Shanshan
%S Proceedings of the Natural Legal Language Processing Workshop 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-494-1
%F akarajaradwong-etal-2026-beyond
%X Free-form legal essay evaluation in NLP treats expert inter-rater stability as a single ceiling number: a correlation score between credentialed examiners is reported and used as the target that an LLM judge should approach. Whether that ceiling is uniform across items has rarely been tested on the expert side. We test it on Thai bar exam essays. Three Bar Council examiners and three anchor LLM judges (Claude 4.6 Opus, Gemini 3.1 Pro, GPT-5.4 Pro) independently graded the same 15 answers from the same four inputs: question, official rubric, gold answer, and student answer. On 10 of the 15 answers, both panels reach α = 0.75. On the remaining 5, drawn from one question where the rubric does not prescribe how to grade a correct answer that omits a decisive citation, the human panel collapses to α = -0.34 while the LLM panel holds at α = 0.68. One examiner’s stricter reading of this underspecified rubric drives the gap. The cluster pattern replicates across seven of eight LLMs that took the exam in Phase 1 and were reused as judges, regardless of their own exam scores: the same model behaves differently in the two roles. The expert stability ceiling is not flat: free-form essay rubrics combine a deterministic correctness element (grounded by the gold answer) with an interpretive quality element (judged without one), and expert disagreement lives entirely on the interpretive element. The bottleneck of free-form essay evaluation is the rubric’s interpretive coverage, not LLM capability.
%U https://aclanthology.org/2026.nllp-1.25/
%P 290-302
Markdown (Informal)
[Beyond the Single-Expert Ceiling: Localizing LLM-as-Judge Stability on Free-Form Legal Essays](https://aclanthology.org/2026.nllp-1.25/) (Akarajaradwong et al., NLLP 2026)
ACL