@inproceedings{zhang-etal-2026-unmasking,
title = "Unmasking Superficial Honesty: Mechanistic Head-Driven Activation Steering for Sycophancy-Free Medical {VLM}s",
author = "Zhang, Mengliang and
Challa, Prajwal and
Gao, Wentao",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1732/",
pages = "39531--39544",
ISBN = "979-8-89176-459-0",
abstract = "Medical Vision-Language Models (VLMs) are vulnerable to misleading clinical prompts and may follow incorrect user assertions instead of image evidence, a behavior commonly referred to as sycophancy. Existing mitigation methods mainly rely on prompt-level constraints to encourage neutral responses. However, through fine-grained integrated gradient analysis, we observe a phenomenon, although prompting strategies can correct the final prediction, the internal multimodal representations remain biased toward text tokens, while visual tokens continue to receive limited attention in late fusion layers. To address this issue, we propose HD-AS (Head-Driven Activation Steering), a training-free inference-time intervention framework. HD-AS identifies sycophancy-related attention heads through activation drift analysis under different prompting conditions. During inference, a lightweight isotropic damping operation is applied through a forward pre-hook on the target projection layers, reducing the dominance of text features in the hidden space and restoring the contribution of visual representations. Experiments on two public medical Visual Question Answering (VQA) datasets show that HD-AS consistently reduces the flip rate while maintaining diagnostic accuracy."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-unmasking">
<titleInfo>
<title>Unmasking Superficial Honesty: Mechanistic Head-Driven Activation Steering for Sycophancy-Free Medical VLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mengliang</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Prajwal</namePart>
<namePart type="family">Challa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wentao</namePart>
<namePart type="family">Gao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Medical Vision-Language Models (VLMs) are vulnerable to misleading clinical prompts and may follow incorrect user assertions instead of image evidence, a behavior commonly referred to as sycophancy. Existing mitigation methods mainly rely on prompt-level constraints to encourage neutral responses. However, through fine-grained integrated gradient analysis, we observe a phenomenon, although prompting strategies can correct the final prediction, the internal multimodal representations remain biased toward text tokens, while visual tokens continue to receive limited attention in late fusion layers. To address this issue, we propose HD-AS (Head-Driven Activation Steering), a training-free inference-time intervention framework. HD-AS identifies sycophancy-related attention heads through activation drift analysis under different prompting conditions. During inference, a lightweight isotropic damping operation is applied through a forward pre-hook on the target projection layers, reducing the dominance of text features in the hidden space and restoring the contribution of visual representations. Experiments on two public medical Visual Question Answering (VQA) datasets show that HD-AS consistently reduces the flip rate while maintaining diagnostic accuracy.</abstract>
<identifier type="citekey">zhang-etal-2026-unmasking</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1732/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>39531</start>
<end>39544</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Unmasking Superficial Honesty: Mechanistic Head-Driven Activation Steering for Sycophancy-Free Medical VLMs
%A Zhang, Mengliang
%A Challa, Prajwal
%A Gao, Wentao
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F zhang-etal-2026-unmasking
%X Medical Vision-Language Models (VLMs) are vulnerable to misleading clinical prompts and may follow incorrect user assertions instead of image evidence, a behavior commonly referred to as sycophancy. Existing mitigation methods mainly rely on prompt-level constraints to encourage neutral responses. However, through fine-grained integrated gradient analysis, we observe a phenomenon, although prompting strategies can correct the final prediction, the internal multimodal representations remain biased toward text tokens, while visual tokens continue to receive limited attention in late fusion layers. To address this issue, we propose HD-AS (Head-Driven Activation Steering), a training-free inference-time intervention framework. HD-AS identifies sycophancy-related attention heads through activation drift analysis under different prompting conditions. During inference, a lightweight isotropic damping operation is applied through a forward pre-hook on the target projection layers, reducing the dominance of text features in the hidden space and restoring the contribution of visual representations. Experiments on two public medical Visual Question Answering (VQA) datasets show that HD-AS consistently reduces the flip rate while maintaining diagnostic accuracy.
%U https://aclanthology.org/2026.emnlp-main.1732/
%P 39531-39544
Markdown (Informal)
[Unmasking Superficial Honesty: Mechanistic Head-Driven Activation Steering for Sycophancy-Free Medical VLMs](https://aclanthology.org/2026.emnlp-main.1732/) (Zhang et al., EMNLP 2026)
ACL