@inproceedings{jamil-etal-2026-learn,
title = "\textit{Learn To Say {NO}!} Incorporating Model Abstention In Medical Vision-Language Models for Pharmacovigilance",
author = "Jamil, Sofia and
Bhat, Chinmay Purushottam and
R, Sooraj Veer and
Saha, Sriparna",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.40/",
pages = "834--848",
ISBN = "979-8-89176-439-2",
abstract = "Abstention is the intentional refusal of models to provide an answer. Adverse drug reactions (ADRs), which range from mild to life-threatening, are often underreported by patients in hospitals and to doctors because patients assume these are unavoidable. With the growing adoption of LLMs and vision language models (VLMs) in healthcare, patients are increasingly reaching out to these models for solutions. In such high-risk scenarios, incorrect or overconfident responses can cause unnecessary anxiety, lead patients to discontinue essential medications, or encourage inappropriate self-medication. In this work, we propose the $\textbf{\textit{MAD-MED}}$, a $\textbf{M}$odel $\textbf{A}$bstention approach for $\textbf{D}$rug Adverse Reaction for $\textbf{MED}$ical-VLMs. It is a training strategy to encourage models to abstain when answering would contradict the available input, exceed the model{'}s knowledge, or violate model ethics. In addition, we release an augmented $\textbf{A}$dverse $\textbf{D}$rug $\textbf{R}$eaction $\textbf{ABS}$tention ($\textit{ADR-ABS}$) dataset comprising image-answerable, text-answerable, and unanswerable questions to support systematic evaluation of abstention behavior. Quantitative and human evaluations confirm that our proposed $\textit{MAD-MED}$ approach improved the abstention capabilities of models considerably while keeping the generative capabilities intact in cases where the model is expected to answer."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="jamil-etal-2026-learn">
<titleInfo>
<title>Learn To Say NO! Incorporating Model Abstention In Medical Vision-Language Models for Pharmacovigilance</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sofia</namePart>
<namePart type="family">Jamil</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chinmay</namePart>
<namePart type="given">Purushottam</namePart>
<namePart type="family">Bhat</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sooraj</namePart>
<namePart type="given">Veer</namePart>
<namePart type="family">R</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sriparna</namePart>
<namePart type="family">Saha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Abstention is the intentional refusal of models to provide an answer. Adverse drug reactions (ADRs), which range from mild to life-threatening, are often underreported by patients in hospitals and to doctors because patients assume these are unavoidable. With the growing adoption of LLMs and vision language models (VLMs) in healthcare, patients are increasingly reaching out to these models for solutions. In such high-risk scenarios, incorrect or overconfident responses can cause unnecessary anxiety, lead patients to discontinue essential medications, or encourage inappropriate self-medication. In this work, we propose the MAD-MED, a Model Abstention approach for Drug Adverse Reaction for MEDical-VLMs. It is a training strategy to encourage models to abstain when answering would contradict the available input, exceed the model’s knowledge, or violate model ethics. In addition, we release an augmented Adverse Drug Reaction ABStention (ADR-ABS) dataset comprising image-answerable, text-answerable, and unanswerable questions to support systematic evaluation of abstention behavior. Quantitative and human evaluations confirm that our proposed MAD-MED approach improved the abstention capabilities of models considerably while keeping the generative capabilities intact in cases where the model is expected to answer.</abstract>
<identifier type="citekey">jamil-etal-2026-learn</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.40/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>834</start>
<end>848</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Learn To Say NO! Incorporating Model Abstention In Medical Vision-Language Models for Pharmacovigilance
%A Jamil, Sofia
%A Bhat, Chinmay Purushottam
%A R, Sooraj Veer
%A Saha, Sriparna
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F jamil-etal-2026-learn
%X Abstention is the intentional refusal of models to provide an answer. Adverse drug reactions (ADRs), which range from mild to life-threatening, are often underreported by patients in hospitals and to doctors because patients assume these are unavoidable. With the growing adoption of LLMs and vision language models (VLMs) in healthcare, patients are increasingly reaching out to these models for solutions. In such high-risk scenarios, incorrect or overconfident responses can cause unnecessary anxiety, lead patients to discontinue essential medications, or encourage inappropriate self-medication. In this work, we propose the MAD-MED, a Model Abstention approach for Drug Adverse Reaction for MEDical-VLMs. It is a training strategy to encourage models to abstain when answering would contradict the available input, exceed the model’s knowledge, or violate model ethics. In addition, we release an augmented Adverse Drug Reaction ABStention (ADR-ABS) dataset comprising image-answerable, text-answerable, and unanswerable questions to support systematic evaluation of abstention behavior. Quantitative and human evaluations confirm that our proposed MAD-MED approach improved the abstention capabilities of models considerably while keeping the generative capabilities intact in cases where the model is expected to answer.
%U https://aclanthology.org/2026.aacl-long.40/
%P 834-848
Markdown (Informal)
[Learn To Say NO! Incorporating Model Abstention In Medical Vision-Language Models for Pharmacovigilance](https://aclanthology.org/2026.aacl-long.40/) (Jamil et al., AACL-IJCNLP 2026)
ACL