@inproceedings{yin-etal-2026-judge,
title = "The Judge Can Debias Itself: Awareness and Self-Debiasing for Reliable {LLM}-as-a-Judge",
author = "Yin, Youbing and
Wu, Jingyu and
Kumar, Anoop and
Novotney, Scott and
Liu, Daben and
Babinsky, Erin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2098/",
doi = "10.18653/2026.findings-emnlp.2098",
pages = "44677--44687",
ISBN = "979-8-89176-462-0",
abstract = "LLM-as-a-Judge systems are increasingly employed for automated evaluation, yet remain susceptible to systematic biases that undermine their reliability. We demonstrate that the judge can effectively debias itself without additional training. We propose two lightweight strategies: bias-aware prompting, which explicitly alerts the judge to potential target biases prior to evaluation, and self-debiasing, in which the judge critiques the initial judgment for bias influence. To broaden the evaluation scope, we introduce three novel benchmarks{---}authority bias (fabricated citations), confidence bias (assertive phrasing), and formatting bias (structured presentation){---}and evaluate across seven bias types in total. Across three frontier models, self-debiasing improves performance in 20 out of 21 model{--}bias combinations with gains up to 15 percentage points, while bias-aware prompting also yields consistent improvements. The recent fine-tuned external critic, however, does not generalize as reliably to the newly-introduced bias types (degradation observed in 5 out of 9 cases). Our findings demonstrate that prompting a judge to reason about bias, either preemptively or retrospectively, is a robust and generalizable approach for enhancing evaluation integrity."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yin-etal-2026-judge">
<titleInfo>
<title>The Judge Can Debias Itself: Awareness and Self-Debiasing for Reliable LLM-as-a-Judge</title>
</titleInfo>
<name type="personal">
<namePart type="given">Youbing</namePart>
<namePart type="family">Yin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingyu</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anoop</namePart>
<namePart type="family">Kumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Scott</namePart>
<namePart type="family">Novotney</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daben</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Erin</namePart>
<namePart type="family">Babinsky</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>LLM-as-a-Judge systems are increasingly employed for automated evaluation, yet remain susceptible to systematic biases that undermine their reliability. We demonstrate that the judge can effectively debias itself without additional training. We propose two lightweight strategies: bias-aware prompting, which explicitly alerts the judge to potential target biases prior to evaluation, and self-debiasing, in which the judge critiques the initial judgment for bias influence. To broaden the evaluation scope, we introduce three novel benchmarks—authority bias (fabricated citations), confidence bias (assertive phrasing), and formatting bias (structured presentation)—and evaluate across seven bias types in total. Across three frontier models, self-debiasing improves performance in 20 out of 21 model–bias combinations with gains up to 15 percentage points, while bias-aware prompting also yields consistent improvements. The recent fine-tuned external critic, however, does not generalize as reliably to the newly-introduced bias types (degradation observed in 5 out of 9 cases). Our findings demonstrate that prompting a judge to reason about bias, either preemptively or retrospectively, is a robust and generalizable approach for enhancing evaluation integrity.</abstract>
<identifier type="citekey">yin-etal-2026-judge</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.2098</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2098/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>44677</start>
<end>44687</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Judge Can Debias Itself: Awareness and Self-Debiasing for Reliable LLM-as-a-Judge
%A Yin, Youbing
%A Wu, Jingyu
%A Kumar, Anoop
%A Novotney, Scott
%A Liu, Daben
%A Babinsky, Erin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F yin-etal-2026-judge
%X LLM-as-a-Judge systems are increasingly employed for automated evaluation, yet remain susceptible to systematic biases that undermine their reliability. We demonstrate that the judge can effectively debias itself without additional training. We propose two lightweight strategies: bias-aware prompting, which explicitly alerts the judge to potential target biases prior to evaluation, and self-debiasing, in which the judge critiques the initial judgment for bias influence. To broaden the evaluation scope, we introduce three novel benchmarks—authority bias (fabricated citations), confidence bias (assertive phrasing), and formatting bias (structured presentation)—and evaluate across seven bias types in total. Across three frontier models, self-debiasing improves performance in 20 out of 21 model–bias combinations with gains up to 15 percentage points, while bias-aware prompting also yields consistent improvements. The recent fine-tuned external critic, however, does not generalize as reliably to the newly-introduced bias types (degradation observed in 5 out of 9 cases). Our findings demonstrate that prompting a judge to reason about bias, either preemptively or retrospectively, is a robust and generalizable approach for enhancing evaluation integrity.
%R 10.18653/2026.findings-emnlp.2098
%U https://aclanthology.org/2026.findings-emnlp.2098/
%U https://doi.org/10.18653/2026.findings-emnlp.2098
%P 44677-44687
Markdown (Informal)
[The Judge Can Debias Itself: Awareness and Self-Debiasing for Reliable LLM-as-a-Judge](https://aclanthology.org/2026.findings-emnlp.2098/) (Yin et al., Findings 2026)
ACL