@inproceedings{sakai-etal-2026-attention,
title = "Attention-Guided Layer Selection for Contrastive Decoding in Large Language Models",
author = "Sakai, Yusuke and
Kertkeidkachorn, Natthawut and
Shirai, Kiyoaki",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.18/",
pages = "351--363",
ISBN = "979-8-89176-439-2",
abstract = "Contrastive decoding methods such as DoLa improve the factuality of Large Language Models (LLMs) by contrasting the output distributions of mature and premature layers. However, DoLa{'}s dynamic layer selection relies solely on divergences in output vocabulary distributions. In this work, we propose three attention-guided strategies{---}Attention-JSD, Attention-Entropy-Max, and Attention-Entropy-Min{---}that leverage structural information carried by internal self-attention mechanisms as a signal for layer selection. Experimental results on TruthfulQA demonstrate that our strategies, particularly Attention-JSD and Attention-Entropy-Min, consistently outperform the original DoLa. We observe significant gains on multi-answer metrics (MC2 and MC3), suggesting that attention distributions can provide a more sensitive signal for resolving factual knowledge than output vocabulary distributions."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sakai-etal-2026-attention">
<titleInfo>
<title>Attention-Guided Layer Selection for Contrastive Decoding in Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yusuke</namePart>
<namePart type="family">Sakai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Natthawut</namePart>
<namePart type="family">Kertkeidkachorn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kiyoaki</namePart>
<namePart type="family">Shirai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Contrastive decoding methods such as DoLa improve the factuality of Large Language Models (LLMs) by contrasting the output distributions of mature and premature layers. However, DoLa’s dynamic layer selection relies solely on divergences in output vocabulary distributions. In this work, we propose three attention-guided strategies—Attention-JSD, Attention-Entropy-Max, and Attention-Entropy-Min—that leverage structural information carried by internal self-attention mechanisms as a signal for layer selection. Experimental results on TruthfulQA demonstrate that our strategies, particularly Attention-JSD and Attention-Entropy-Min, consistently outperform the original DoLa. We observe significant gains on multi-answer metrics (MC2 and MC3), suggesting that attention distributions can provide a more sensitive signal for resolving factual knowledge than output vocabulary distributions.</abstract>
<identifier type="citekey">sakai-etal-2026-attention</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.18/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>351</start>
<end>363</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Attention-Guided Layer Selection for Contrastive Decoding in Large Language Models
%A Sakai, Yusuke
%A Kertkeidkachorn, Natthawut
%A Shirai, Kiyoaki
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F sakai-etal-2026-attention
%X Contrastive decoding methods such as DoLa improve the factuality of Large Language Models (LLMs) by contrasting the output distributions of mature and premature layers. However, DoLa’s dynamic layer selection relies solely on divergences in output vocabulary distributions. In this work, we propose three attention-guided strategies—Attention-JSD, Attention-Entropy-Max, and Attention-Entropy-Min—that leverage structural information carried by internal self-attention mechanisms as a signal for layer selection. Experimental results on TruthfulQA demonstrate that our strategies, particularly Attention-JSD and Attention-Entropy-Min, consistently outperform the original DoLa. We observe significant gains on multi-answer metrics (MC2 and MC3), suggesting that attention distributions can provide a more sensitive signal for resolving factual knowledge than output vocabulary distributions.
%U https://aclanthology.org/2026.aacl-long.18/
%P 351-363
Markdown (Informal)
[Attention-Guided Layer Selection for Contrastive Decoding in Large Language Models](https://aclanthology.org/2026.aacl-long.18/) (Sakai et al., AACL-IJCNLP 2026)
ACL
- Yusuke Sakai, Natthawut Kertkeidkachorn, and Kiyoaki Shirai. 2026. Attention-Guided Layer Selection for Contrastive Decoding in Large Language Models. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 351–363, Hengqin, China. Association for Computational Linguistics.