@inproceedings{wong-etal-2026-semantic,
title = "Semantic Distractor Robustness in {LLM}s: Exploring the Underlying Mechanisms",
author = "Wong, Xiangyu and
Yu, Eugene J. and
Xiong, Chenhao and
Li, Zheng and
Song, Yifan and
Li, Sujian",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1573/",
pages = "35820--35841",
ISBN = "979-8-89176-459-0",
abstract = "Large language models drop in performance when their context contains $\textit{semantic distractors}$: context topically related to the question but off the reasoning chain. Prior mechanistic work tracks a single injected distractor token, missing failures in which the model predicts other context tokens or out-of-context but topically related ones. We instead track the model{'}s actual top non-gold prediction, providing a localization target for every observed top-1 failure in the analyzed subset. Failures converge on a small set of mid-to-late-layer $\textit{answer-selection heads}$ (AS-Heads): patching their clean outputs into distracting runs recovers 60-$86\\%$ of failed predictions across four base models from two families (random heads recover $\leq 8\\%$), with the mechanism also replicating in two instruction-tuned models. Clean-prompt ablations indicate that these heads also contribute to normal answer selection, so their role is not specific to distractors. Redirecting their attention onto an entity span causes the model to transcribe that span at $22$-$83\\%$ (vs. $\leq 7.4\\%$ for random heads), establishing a causally consequential intervention point. The localized heads transfer across MuSiQue, HotpotQA, 2WikiMultiHopQA, and non-Wikipedia RAG settings. At inference, their attention provides a training-free context-filtering signal that lifts F1 by up to $+35$ points on disrupted samples and is competitive with retrieval-head methods.$"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wong-etal-2026-semantic">
<titleInfo>
<title>Semantic Distractor Robustness in LLMs: Exploring the Underlying Mechanisms</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xiangyu</namePart>
<namePart type="family">Wong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Eugene</namePart>
<namePart type="given">J</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chenhao</namePart>
<namePart type="family">Xiong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zheng</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yifan</namePart>
<namePart type="family">Song</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sujian</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large language models drop in performance when their context contains semantic distractors: context topically related to the question but off the reasoning chain. Prior mechanistic work tracks a single injected distractor token, missing failures in which the model predicts other context tokens or out-of-context but topically related ones. We instead track the model’s actual top non-gold prediction, providing a localization target for every observed top-1 failure in the analyzed subset. Failures converge on a small set of mid-to-late-layer answer-selection heads (AS-Heads): patching their clean outputs into distracting runs recovers 60-86\% of failed predictions across four base models from two families (random heads recover łeq 8\%), with the mechanism also replicating in two instruction-tuned models. Clean-prompt ablations indicate that these heads also contribute to normal answer selection, so their role is not specific to distractors. Redirecting their attention onto an entity span causes the model to transcribe that span at 22-83\% (vs. łeq 7.4\% for random heads), establishing a causally consequential intervention point. The localized heads transfer across MuSiQue, HotpotQA, 2WikiMultiHopQA, and non-Wikipedia RAG settings. At inference, their attention provides a training-free context-filtering signal that lifts F1 by up to +35 points on disrupted samples and is competitive with retrieval-head methods.</abstract>
<identifier type="citekey">wong-etal-2026-semantic</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1573/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>35820</start>
<end>35841</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Semantic Distractor Robustness in LLMs: Exploring the Underlying Mechanisms
%A Wong, Xiangyu
%A Yu, Eugene J.
%A Xiong, Chenhao
%A Li, Zheng
%A Song, Yifan
%A Li, Sujian
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F wong-etal-2026-semantic
%X Large language models drop in performance when their context contains semantic distractors: context topically related to the question but off the reasoning chain. Prior mechanistic work tracks a single injected distractor token, missing failures in which the model predicts other context tokens or out-of-context but topically related ones. We instead track the model’s actual top non-gold prediction, providing a localization target for every observed top-1 failure in the analyzed subset. Failures converge on a small set of mid-to-late-layer answer-selection heads (AS-Heads): patching their clean outputs into distracting runs recovers 60-86\% of failed predictions across four base models from two families (random heads recover łeq 8\%), with the mechanism also replicating in two instruction-tuned models. Clean-prompt ablations indicate that these heads also contribute to normal answer selection, so their role is not specific to distractors. Redirecting their attention onto an entity span causes the model to transcribe that span at 22-83\% (vs. łeq 7.4\% for random heads), establishing a causally consequential intervention point. The localized heads transfer across MuSiQue, HotpotQA, 2WikiMultiHopQA, and non-Wikipedia RAG settings. At inference, their attention provides a training-free context-filtering signal that lifts F1 by up to +35 points on disrupted samples and is competitive with retrieval-head methods.
%U https://aclanthology.org/2026.emnlp-main.1573/
%P 35820-35841
Markdown (Informal)
[Semantic Distractor Robustness in LLMs: Exploring the Underlying Mechanisms](https://aclanthology.org/2026.emnlp-main.1573/) (Wong et al., EMNLP 2026)
ACL