@inproceedings{tang-etal-2026-probe,
title = "Probe and Steer: Localizing and Mitigating Position Bias in Pairwise {LLM}-as-a-Judge",
author = "Tang, Yulu and
Kang, Chun and
Zhou, Yangfan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1949/",
pages = "41549--41565",
ISBN = "979-8-89176-462-0",
abstract = "LLM-as-a-Judge has emerged as a scalable alternative to costly human evaluation. However, LLM judges suffer from position bias in pairwise evaluation: the judge systematically favors the first or second candidate regardless of content, forcing practitioners to perform bidirectional evaluation at twice the inference cost. We propose Probe-and-Steer (PaS), a probe-localized, single-layer steering method for mitigating position bias. PaS localizes the bias-critical layer via a layer-wise logit-difference probe and trains a single residual-stream steering vector at that layer under a multi-objective loss, requiring minimal labeled data while keeping the LLM fully frozen. Through layer-wise probing across multiple model families, we uncover three qualitatively distinct bias-distribution patterns and delineate a clear applicability boundary for residual-stream steering. Experiments on an XSum-based dataset with Llama-3.1-8B-Instruct show that PaS delivers the largest gains among all compared methods, improving consistency by 22.9 pp and accuracy by 16.4 pp while shrinking positional asymmetry by $33\times$ (from 0.471 to 0.014), adding the lowest inference overhead at 4{\%}."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="tang-etal-2026-probe">
<titleInfo>
<title>Probe and Steer: Localizing and Mitigating Position Bias in Pairwise LLM-as-a-Judge</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yulu</namePart>
<namePart type="family">Tang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chun</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yangfan</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>LLM-as-a-Judge has emerged as a scalable alternative to costly human evaluation. However, LLM judges suffer from position bias in pairwise evaluation: the judge systematically favors the first or second candidate regardless of content, forcing practitioners to perform bidirectional evaluation at twice the inference cost. We propose Probe-and-Steer (PaS), a probe-localized, single-layer steering method for mitigating position bias. PaS localizes the bias-critical layer via a layer-wise logit-difference probe and trains a single residual-stream steering vector at that layer under a multi-objective loss, requiring minimal labeled data while keeping the LLM fully frozen. Through layer-wise probing across multiple model families, we uncover three qualitatively distinct bias-distribution patterns and delineate a clear applicability boundary for residual-stream steering. Experiments on an XSum-based dataset with Llama-3.1-8B-Instruct show that PaS delivers the largest gains among all compared methods, improving consistency by 22.9 pp and accuracy by 16.4 pp while shrinking positional asymmetry by 33\times (from 0.471 to 0.014), adding the lowest inference overhead at 4%.</abstract>
<identifier type="citekey">tang-etal-2026-probe</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1949/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>41549</start>
<end>41565</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Probe and Steer: Localizing and Mitigating Position Bias in Pairwise LLM-as-a-Judge
%A Tang, Yulu
%A Kang, Chun
%A Zhou, Yangfan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F tang-etal-2026-probe
%X LLM-as-a-Judge has emerged as a scalable alternative to costly human evaluation. However, LLM judges suffer from position bias in pairwise evaluation: the judge systematically favors the first or second candidate regardless of content, forcing practitioners to perform bidirectional evaluation at twice the inference cost. We propose Probe-and-Steer (PaS), a probe-localized, single-layer steering method for mitigating position bias. PaS localizes the bias-critical layer via a layer-wise logit-difference probe and trains a single residual-stream steering vector at that layer under a multi-objective loss, requiring minimal labeled data while keeping the LLM fully frozen. Through layer-wise probing across multiple model families, we uncover three qualitatively distinct bias-distribution patterns and delineate a clear applicability boundary for residual-stream steering. Experiments on an XSum-based dataset with Llama-3.1-8B-Instruct show that PaS delivers the largest gains among all compared methods, improving consistency by 22.9 pp and accuracy by 16.4 pp while shrinking positional asymmetry by 33\times (from 0.471 to 0.014), adding the lowest inference overhead at 4%.
%U https://aclanthology.org/2026.findings-emnlp.1949/
%P 41549-41565
Markdown (Informal)
[Probe and Steer: Localizing and Mitigating Position Bias in Pairwise LLM-as-a-Judge](https://aclanthology.org/2026.findings-emnlp.1949/) (Tang et al., Findings 2026)
ACL