@inproceedings{sim-etal-2026-model,
title = "The Model Knows Where to Look: Visual-Grounding-Guided Head Suppression for Mitigating Hallucination in {LVLM}s",
author = "Sim, Mong Yuan and
Zhang, Wei Emma and
Dai, Xiang and
Fang, Byron",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.152/",
pages = "3347--3367",
ISBN = "979-8-89176-439-2",
abstract = "Large Vision-Language Models (LVLMs) achieve strong multimodal reasoning capabilities, yet are still prone to object hallucination - often generating content that is not grounded by the visual input. Existing attention-based training-free mitigating methods rely on visual attention heads identified by high attention weights or sensitivity to visual token perturbations. However, we show that these identified visual attention heads are not necessarily responsible for visual grounding, and using this as guidance is suboptimal. Motivated by this observation, we propose a novel training-free decoding method that identifies visually sensitive heads by counterfactual region masking. We find that these heads provide a clear signal when visual grounding is required and can therefore be used to mitigate hallucination by guiding the suppression of hallucination heads. Across multiple LVLMs and benchmark image captioning and VQA datasets, our method achieves best or near-best performance in reducing hallucination while preserving generation quality."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sim-etal-2026-model">
<titleInfo>
<title>The Model Knows Where to Look: Visual-Grounding-Guided Head Suppression for Mitigating Hallucination in LVLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mong</namePart>
<namePart type="given">Yuan</namePart>
<namePart type="family">Sim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="given">Emma</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiang</namePart>
<namePart type="family">Dai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Byron</namePart>
<namePart type="family">Fang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Large Vision-Language Models (LVLMs) achieve strong multimodal reasoning capabilities, yet are still prone to object hallucination - often generating content that is not grounded by the visual input. Existing attention-based training-free mitigating methods rely on visual attention heads identified by high attention weights or sensitivity to visual token perturbations. However, we show that these identified visual attention heads are not necessarily responsible for visual grounding, and using this as guidance is suboptimal. Motivated by this observation, we propose a novel training-free decoding method that identifies visually sensitive heads by counterfactual region masking. We find that these heads provide a clear signal when visual grounding is required and can therefore be used to mitigate hallucination by guiding the suppression of hallucination heads. Across multiple LVLMs and benchmark image captioning and VQA datasets, our method achieves best or near-best performance in reducing hallucination while preserving generation quality.</abstract>
<identifier type="citekey">sim-etal-2026-model</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.152/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>3347</start>
<end>3367</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Model Knows Where to Look: Visual-Grounding-Guided Head Suppression for Mitigating Hallucination in LVLMs
%A Sim, Mong Yuan
%A Zhang, Wei Emma
%A Dai, Xiang
%A Fang, Byron
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F sim-etal-2026-model
%X Large Vision-Language Models (LVLMs) achieve strong multimodal reasoning capabilities, yet are still prone to object hallucination - often generating content that is not grounded by the visual input. Existing attention-based training-free mitigating methods rely on visual attention heads identified by high attention weights or sensitivity to visual token perturbations. However, we show that these identified visual attention heads are not necessarily responsible for visual grounding, and using this as guidance is suboptimal. Motivated by this observation, we propose a novel training-free decoding method that identifies visually sensitive heads by counterfactual region masking. We find that these heads provide a clear signal when visual grounding is required and can therefore be used to mitigate hallucination by guiding the suppression of hallucination heads. Across multiple LVLMs and benchmark image captioning and VQA datasets, our method achieves best or near-best performance in reducing hallucination while preserving generation quality.
%U https://aclanthology.org/2026.aacl-long.152/
%P 3347-3367
Markdown (Informal)
[The Model Knows Where to Look: Visual-Grounding-Guided Head Suppression for Mitigating Hallucination in LVLMs](https://aclanthology.org/2026.aacl-long.152/) (Sim et al., AACL-IJCNLP 2026)
ACL