@article{sun-etal-2026-evaluation,
title = "Evaluation Framework for Highlight Explanations of Context Utilization in Language Models",
author = "Sun, Jingyi and
Atanasova, Pepa and
Choudhury, Sagnik Ray and
Islam, Sekh Mainul and
Augenstein, Isabelle",
journal = "Computational Linguistics",
volume = "52",
number = "3",
month = sep,
year = "2026",
address = "Cambridge, MA",
publisher = "MIT Press",
url = "https://aclanthology.org/2026.cl-3.8/",
doi = "10.1162/coli.a.621",
pages = "1207--1241",
abstract = "Context utilization, the ability of Language Models (LMs) to incorporate relevant information from the provided context when generating responses, remains largely opaque to users, who cannot determine whether models draw from parametric memory or provided context, nor identify which specific context pieces inform the response. Highlight explanations (HEs) offer a natural solution as they can point to the exact context pieces and tokens that influenced model outputs. However, no existing work evaluates their effectiveness in accurately explaining context utilization. We address this gap by introducing the first gold standard HE evaluation framework for context attribution, using controlled test cases with known ground-truth context usage, thereby avoiding the limitations of existing indirect proxy evaluations. To demonstrate the framework{'}s broad applicability, we evaluate four HE methods{---}three established techniques and MechLight, a mechanistic interpretability approach we adapt for this task{---}across four context scenarios, four datasets, and five LMs. Overall, we find that MechLight performs best across all context scenarios. However, our findings reveal systematic failures in all methods: Explanation accuracy degrades significantly with context length, and all methods exhibit strong positional biases in multi-document settings. Surprisingly, widely used gradient-based methods provide little value for understanding context usage. These results challenge HEs' utility in retrieval-augmented generation, factual verification, and other applications. Our framework provides the foundation for developing accurate context attribution methods."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sun-etal-2026-evaluation">
<titleInfo>
<title>Evaluation Framework for Highlight Explanations of Context Utilization in Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jingyi</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sagnik</namePart>
<namePart type="given">Ray</namePart>
<namePart type="family">Choudhury</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sekh</namePart>
<namePart type="given">Mainul</namePart>
<namePart type="family">Islam</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Isabelle</namePart>
<namePart type="family">Augenstein</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-09</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<genre authority="bibutilsgt">journal article</genre>
<relatedItem type="host">
<titleInfo>
<title>Computational Linguistics</title>
</titleInfo>
<originInfo>
<issuance>continuing</issuance>
<publisher>MIT Press</publisher>
<place>
<placeTerm type="text">Cambridge, MA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">periodical</genre>
<genre authority="bibutilsgt">academic journal</genre>
</relatedItem>
<abstract>Context utilization, the ability of Language Models (LMs) to incorporate relevant information from the provided context when generating responses, remains largely opaque to users, who cannot determine whether models draw from parametric memory or provided context, nor identify which specific context pieces inform the response. Highlight explanations (HEs) offer a natural solution as they can point to the exact context pieces and tokens that influenced model outputs. However, no existing work evaluates their effectiveness in accurately explaining context utilization. We address this gap by introducing the first gold standard HE evaluation framework for context attribution, using controlled test cases with known ground-truth context usage, thereby avoiding the limitations of existing indirect proxy evaluations. To demonstrate the framework’s broad applicability, we evaluate four HE methods—three established techniques and MechLight, a mechanistic interpretability approach we adapt for this task—across four context scenarios, four datasets, and five LMs. Overall, we find that MechLight performs best across all context scenarios. However, our findings reveal systematic failures in all methods: Explanation accuracy degrades significantly with context length, and all methods exhibit strong positional biases in multi-document settings. Surprisingly, widely used gradient-based methods provide little value for understanding context usage. These results challenge HEs’ utility in retrieval-augmented generation, factual verification, and other applications. Our framework provides the foundation for developing accurate context attribution methods.</abstract>
<identifier type="citekey">sun-etal-2026-evaluation</identifier>
<identifier type="doi">10.1162/coli.a.621</identifier>
<location>
<url>https://aclanthology.org/2026.cl-3.8/</url>
</location>
<part>
<date>2026-09</date>
<detail type="volume"><number>52</number></detail>
<detail type="issue"><number>3</number></detail>
<extent unit="page">
<start>1207</start>
<end>1241</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Journal Article
%T Evaluation Framework for Highlight Explanations of Context Utilization in Language Models
%A Sun, Jingyi
%A Atanasova, Pepa
%A Choudhury, Sagnik Ray
%A Islam, Sekh Mainul
%A Augenstein, Isabelle
%J Computational Linguistics
%D 2026
%8 September
%V 52
%N 3
%I MIT Press
%C Cambridge, MA
%F sun-etal-2026-evaluation
%X Context utilization, the ability of Language Models (LMs) to incorporate relevant information from the provided context when generating responses, remains largely opaque to users, who cannot determine whether models draw from parametric memory or provided context, nor identify which specific context pieces inform the response. Highlight explanations (HEs) offer a natural solution as they can point to the exact context pieces and tokens that influenced model outputs. However, no existing work evaluates their effectiveness in accurately explaining context utilization. We address this gap by introducing the first gold standard HE evaluation framework for context attribution, using controlled test cases with known ground-truth context usage, thereby avoiding the limitations of existing indirect proxy evaluations. To demonstrate the framework’s broad applicability, we evaluate four HE methods—three established techniques and MechLight, a mechanistic interpretability approach we adapt for this task—across four context scenarios, four datasets, and five LMs. Overall, we find that MechLight performs best across all context scenarios. However, our findings reveal systematic failures in all methods: Explanation accuracy degrades significantly with context length, and all methods exhibit strong positional biases in multi-document settings. Surprisingly, widely used gradient-based methods provide little value for understanding context usage. These results challenge HEs’ utility in retrieval-augmented generation, factual verification, and other applications. Our framework provides the foundation for developing accurate context attribution methods.
%R 10.1162/coli.a.621
%U https://aclanthology.org/2026.cl-3.8/
%U https://doi.org/10.1162/coli.a.621
%P 1207-1241
Markdown (Informal)
[Evaluation Framework for Highlight Explanations of Context Utilization in Language Models](https://aclanthology.org/2026.cl-3.8/) (Sun et al., CL 2026)
ACL