@inproceedings{bhaskar-etal-2026-fbhm,
title = "{FBHM}: Functional Benchmarking and Steering of {VLM}s for Hateful Meme Detection",
author = "Bhaskar, Paramananda and
Rizwan, Naquee and
Jogchand, Daksh and
Pandey, Saurabh Kumar and
Mukherjee, Animesh",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1596/",
pages = "36273--36297",
ISBN = "979-8-89176-459-0",
abstract = "Hateful meme detection remains a formidable challenge for vision-language models, as existing benchmarks are structurally observational{---}confounding rhetorical hate mechanisms with target community features and preventing causal evaluation of model vulnerabilities. To address this, we introduce **FBHM**, a systematically curated benchmark of ***F****unctionality* ***B****ased* ***H****ateful* ***M****emes* constructed along two orthogonal axes: 25 distinct rhetorical functionalities and 10 target communities (5,000 memes total). Benchmarking state-of-the-art VLMs reveals a severe generalization gap: models highly accurate on standard datasets catastrophically drop to near-random performance on **FBHM**, proving they exploit dataset-specific heuristics rather than robust multimodal reasoning. To efficiently close this gap, we propose **LSV** (***l****earnable* ***s****t*eering ***v****ectors*), an ultra-low data regime strategy that applies a causal intervention objective on as few as 500 steering samples (50 unique base memes), boosting **FBHM** performance by {\textasciitilde}30 Macro-F1 points while outperforming in-context learning and PEFT without degrading source-domain performance."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="bhaskar-etal-2026-fbhm">
<titleInfo>
<title>FBHM: Functional Benchmarking and Steering of VLMs for Hateful Meme Detection</title>
</titleInfo>
<name type="personal">
<namePart type="given">Paramananda</namePart>
<namePart type="family">Bhaskar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Naquee</namePart>
<namePart type="family">Rizwan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daksh</namePart>
<namePart type="family">Jogchand</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Saurabh</namePart>
<namePart type="given">Kumar</namePart>
<namePart type="family">Pandey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Animesh</namePart>
<namePart type="family">Mukherjee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Hateful meme detection remains a formidable challenge for vision-language models, as existing benchmarks are structurally observational—confounding rhetorical hate mechanisms with target community features and preventing causal evaluation of model vulnerabilities. To address this, we introduce **FBHM**, a systematically curated benchmark of ***F****unctionality* ***B****ased* ***H****ateful* ***M****emes* constructed along two orthogonal axes: 25 distinct rhetorical functionalities and 10 target communities (5,000 memes total). Benchmarking state-of-the-art VLMs reveals a severe generalization gap: models highly accurate on standard datasets catastrophically drop to near-random performance on **FBHM**, proving they exploit dataset-specific heuristics rather than robust multimodal reasoning. To efficiently close this gap, we propose **LSV** (***l****earnable* ***s****t*eering ***v****ectors*), an ultra-low data regime strategy that applies a causal intervention objective on as few as 500 steering samples (50 unique base memes), boosting **FBHM** performance by ~30 Macro-F1 points while outperforming in-context learning and PEFT without degrading source-domain performance.</abstract>
<identifier type="citekey">bhaskar-etal-2026-fbhm</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1596/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>36273</start>
<end>36297</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T FBHM: Functional Benchmarking and Steering of VLMs for Hateful Meme Detection
%A Bhaskar, Paramananda
%A Rizwan, Naquee
%A Jogchand, Daksh
%A Pandey, Saurabh Kumar
%A Mukherjee, Animesh
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F bhaskar-etal-2026-fbhm
%X Hateful meme detection remains a formidable challenge for vision-language models, as existing benchmarks are structurally observational—confounding rhetorical hate mechanisms with target community features and preventing causal evaluation of model vulnerabilities. To address this, we introduce **FBHM**, a systematically curated benchmark of ***F****unctionality* ***B****ased* ***H****ateful* ***M****emes* constructed along two orthogonal axes: 25 distinct rhetorical functionalities and 10 target communities (5,000 memes total). Benchmarking state-of-the-art VLMs reveals a severe generalization gap: models highly accurate on standard datasets catastrophically drop to near-random performance on **FBHM**, proving they exploit dataset-specific heuristics rather than robust multimodal reasoning. To efficiently close this gap, we propose **LSV** (***l****earnable* ***s****t*eering ***v****ectors*), an ultra-low data regime strategy that applies a causal intervention objective on as few as 500 steering samples (50 unique base memes), boosting **FBHM** performance by ~30 Macro-F1 points while outperforming in-context learning and PEFT without degrading source-domain performance.
%U https://aclanthology.org/2026.emnlp-main.1596/
%P 36273-36297
Markdown (Informal)
[FBHM: Functional Benchmarking and Steering of VLMs for Hateful Meme Detection](https://aclanthology.org/2026.emnlp-main.1596/) (Bhaskar et al., EMNLP 2026)
ACL