@inproceedings{tan-etal-2026-contrastive,
title = "Contrastive Attribution in the Wild: An Interpretability Analysis of {LLM} Failures on Realistic Benchmarks",
author = "Tan, Rongyuan and
Zhang, Jue and
Li, Zhuozhao and
Lin, Qingwei and
Rajmohan, Saravan and
Zhang, Dongmei",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2003/",
pages = "45618--45649",
ISBN = "979-8-89176-459-0",
abstract = "Interpretability methods are increasingly invoked to diagnose Large Language Model (LLM) failures, but most evidence comes from short prompts or toy templates, leaving their utility on practical benchmarks unclear. We cast failure diagnosis as contrastive attribution over target{--}contrast token pairs, attributing the logit gap between incorrect and correct output tokens to input tokens and internal states, and introduce a batch-packed multi-target backpropagation scheme that scales cross-layer attribution graphs to long-context inputs. Instantiated with LRP-style attribution, the framework is evaluated on IFEval, GAIA2, MATH, and EvalPlus across multiple model sizes and post-training checkpoints. Along the four axes of coverage, visibility, reliability, and dynamics, contrastive attribution surfaces recurring failure modes such as underweighting task-relevant tokens and overweighting distractors, demonstrating its practical value as a scalable diagnostic tool for realistic LLM failures."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="tan-etal-2026-contrastive">
<titleInfo>
<title>Contrastive Attribution in the Wild: An Interpretability Analysis of LLM Failures on Realistic Benchmarks</title>
</titleInfo>
<name type="personal">
<namePart type="given">Rongyuan</namePart>
<namePart type="family">Tan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhuozhao</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qingwei</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Saravan</namePart>
<namePart type="family">Rajmohan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dongmei</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Interpretability methods are increasingly invoked to diagnose Large Language Model (LLM) failures, but most evidence comes from short prompts or toy templates, leaving their utility on practical benchmarks unclear. We cast failure diagnosis as contrastive attribution over target–contrast token pairs, attributing the logit gap between incorrect and correct output tokens to input tokens and internal states, and introduce a batch-packed multi-target backpropagation scheme that scales cross-layer attribution graphs to long-context inputs. Instantiated with LRP-style attribution, the framework is evaluated on IFEval, GAIA2, MATH, and EvalPlus across multiple model sizes and post-training checkpoints. Along the four axes of coverage, visibility, reliability, and dynamics, contrastive attribution surfaces recurring failure modes such as underweighting task-relevant tokens and overweighting distractors, demonstrating its practical value as a scalable diagnostic tool for realistic LLM failures.</abstract>
<identifier type="citekey">tan-etal-2026-contrastive</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2003/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>45618</start>
<end>45649</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Contrastive Attribution in the Wild: An Interpretability Analysis of LLM Failures on Realistic Benchmarks
%A Tan, Rongyuan
%A Zhang, Jue
%A Li, Zhuozhao
%A Lin, Qingwei
%A Rajmohan, Saravan
%A Zhang, Dongmei
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F tan-etal-2026-contrastive
%X Interpretability methods are increasingly invoked to diagnose Large Language Model (LLM) failures, but most evidence comes from short prompts or toy templates, leaving their utility on practical benchmarks unclear. We cast failure diagnosis as contrastive attribution over target–contrast token pairs, attributing the logit gap between incorrect and correct output tokens to input tokens and internal states, and introduce a batch-packed multi-target backpropagation scheme that scales cross-layer attribution graphs to long-context inputs. Instantiated with LRP-style attribution, the framework is evaluated on IFEval, GAIA2, MATH, and EvalPlus across multiple model sizes and post-training checkpoints. Along the four axes of coverage, visibility, reliability, and dynamics, contrastive attribution surfaces recurring failure modes such as underweighting task-relevant tokens and overweighting distractors, demonstrating its practical value as a scalable diagnostic tool for realistic LLM failures.
%U https://aclanthology.org/2026.emnlp-main.2003/
%P 45618-45649
Markdown (Informal)
[Contrastive Attribution in the Wild: An Interpretability Analysis of LLM Failures on Realistic Benchmarks](https://aclanthology.org/2026.emnlp-main.2003/) (Tan et al., EMNLP 2026)
ACL