@inproceedings{lee-lee-2026-beyond,
title = "Beyond Output-Level Correction: Tracing Positional Bias to Early-Layer Commitment in Transformer Multiple-Choice Evaluation",
author = "Lee, Soyeon and
Lee, Minhyeok",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1059/",
pages = "24307--24328",
ISBN = "979-8-89176-459-0",
abstract = "Language models exhibit positional bias on multiple-choice benchmarks, scoring the same question differently depending on which position holds the correct answer. Existing work measures the bias at the output and corrects it by shuffling option orders, but does not explain where in the network it originates. Using causal activation patching with a formalized Average Indirect Effect (AIE) metric, we trace the layer-by-layer formation of positional commitment across three model families and benchmarks. Positive causal mass concentrates in early-to-mid layers (normalized depth below 0.60) for the higher-capacity models tested, with Pearson correlations above 0.89 across nine model-dataset conditions and three output metrics, and the localization is stable under multi-layer windowed patching. A controlled factorization that permutes option labels and option order independently shows this profile tracks the label-identity component of the bias rather than generic order sensitivity. Sublayer decomposition reveals MLP-dominated early layers (about 59{\%}) and attention-dominated mid-layers (about 76{\%}), consistent with label information being integrated through attention after semantic content is encoded by MLPs. Negative controls show no detectable early-layer separation between robustly correct and bias-manifesting questions, supporting a shared answer-commitment pathway rather than a dedicated bias circuit and explaining why output-level shuffling stabilizes scores without altering internal preference."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lee-lee-2026-beyond">
<titleInfo>
<title>Beyond Output-Level Correction: Tracing Positional Bias to Early-Layer Commitment in Transformer Multiple-Choice Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Soyeon</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minhyeok</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Language models exhibit positional bias on multiple-choice benchmarks, scoring the same question differently depending on which position holds the correct answer. Existing work measures the bias at the output and corrects it by shuffling option orders, but does not explain where in the network it originates. Using causal activation patching with a formalized Average Indirect Effect (AIE) metric, we trace the layer-by-layer formation of positional commitment across three model families and benchmarks. Positive causal mass concentrates in early-to-mid layers (normalized depth below 0.60) for the higher-capacity models tested, with Pearson correlations above 0.89 across nine model-dataset conditions and three output metrics, and the localization is stable under multi-layer windowed patching. A controlled factorization that permutes option labels and option order independently shows this profile tracks the label-identity component of the bias rather than generic order sensitivity. Sublayer decomposition reveals MLP-dominated early layers (about 59%) and attention-dominated mid-layers (about 76%), consistent with label information being integrated through attention after semantic content is encoded by MLPs. Negative controls show no detectable early-layer separation between robustly correct and bias-manifesting questions, supporting a shared answer-commitment pathway rather than a dedicated bias circuit and explaining why output-level shuffling stabilizes scores without altering internal preference.</abstract>
<identifier type="citekey">lee-lee-2026-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1059/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>24307</start>
<end>24328</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Output-Level Correction: Tracing Positional Bias to Early-Layer Commitment in Transformer Multiple-Choice Evaluation
%A Lee, Soyeon
%A Lee, Minhyeok
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F lee-lee-2026-beyond
%X Language models exhibit positional bias on multiple-choice benchmarks, scoring the same question differently depending on which position holds the correct answer. Existing work measures the bias at the output and corrects it by shuffling option orders, but does not explain where in the network it originates. Using causal activation patching with a formalized Average Indirect Effect (AIE) metric, we trace the layer-by-layer formation of positional commitment across three model families and benchmarks. Positive causal mass concentrates in early-to-mid layers (normalized depth below 0.60) for the higher-capacity models tested, with Pearson correlations above 0.89 across nine model-dataset conditions and three output metrics, and the localization is stable under multi-layer windowed patching. A controlled factorization that permutes option labels and option order independently shows this profile tracks the label-identity component of the bias rather than generic order sensitivity. Sublayer decomposition reveals MLP-dominated early layers (about 59%) and attention-dominated mid-layers (about 76%), consistent with label information being integrated through attention after semantic content is encoded by MLPs. Negative controls show no detectable early-layer separation between robustly correct and bias-manifesting questions, supporting a shared answer-commitment pathway rather than a dedicated bias circuit and explaining why output-level shuffling stabilizes scores without altering internal preference.
%U https://aclanthology.org/2026.emnlp-main.1059/
%P 24307-24328
Markdown (Informal)
[Beyond Output-Level Correction: Tracing Positional Bias to Early-Layer Commitment in Transformer Multiple-Choice Evaluation](https://aclanthology.org/2026.emnlp-main.1059/) (Lee & Lee, EMNLP 2026)
ACL