@inproceedings{yao-etal-2026-multimodal,
title = "Multimodal Reward Hacking in Reinforcement Learning",
author = "Yao, Jiayu and
Wang, Yiwei and
Zhang, Anmeng and
Sun, Zhe and
Wang, Songsong and
Mei, Lingrui and
Ge, Yuyao and
Liu, Shenghua",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.980/",
pages = "20718--20733",
ISBN = "979-8-89176-462-0",
abstract = "Reinforcement learning is increasingly used to align multimodal large language models (MLLMs). However, higher proxy rewards do not necessarily lead to better task performance, especially when visual evidence is difficult to verify with text-only or weakly grounded reward signals. We systematically study reward hacking in MLLM reinforcement learning by varying reward design, data ambiguity, model scale, and RL algorithm. To separate RL-induced failures from pre-existing weaknesses, we introduce the Newly Rewarded Failure Rate (NRFR), which measures the hacking rate among samples where RL achieves higher proxy reward than the SFT baseline. Our analysis yields three main findings. First, RL can actively create new failures rather than merely amplify existing ones, and the effect is especially pronounced near ambiguous decision boundaries. Second, robustness depends jointly on model scale, reward design, and RL algorithm, and scaling mitigates hacking but cannot replace reliable rewards. Third, incorporating visual evidence helps only when verification is reliable, since keyword-based verification introduces additional exploitation opportunities whereas VLM-based semantic verification can reduce reward hacking. Overall, multimodal reward hacking is a systematic consequence of optimizing imperfect reward signals, highlighting the need for rewards and verifiers that remain reliable under optimization pressure."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yao-etal-2026-multimodal">
<titleInfo>
<title>Multimodal Reward Hacking in Reinforcement Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jiayu</namePart>
<namePart type="family">Yao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yiwei</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anmeng</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhe</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Songsong</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lingrui</namePart>
<namePart type="family">Mei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuyao</namePart>
<namePart type="family">Ge</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shenghua</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Reinforcement learning is increasingly used to align multimodal large language models (MLLMs). However, higher proxy rewards do not necessarily lead to better task performance, especially when visual evidence is difficult to verify with text-only or weakly grounded reward signals. We systematically study reward hacking in MLLM reinforcement learning by varying reward design, data ambiguity, model scale, and RL algorithm. To separate RL-induced failures from pre-existing weaknesses, we introduce the Newly Rewarded Failure Rate (NRFR), which measures the hacking rate among samples where RL achieves higher proxy reward than the SFT baseline. Our analysis yields three main findings. First, RL can actively create new failures rather than merely amplify existing ones, and the effect is especially pronounced near ambiguous decision boundaries. Second, robustness depends jointly on model scale, reward design, and RL algorithm, and scaling mitigates hacking but cannot replace reliable rewards. Third, incorporating visual evidence helps only when verification is reliable, since keyword-based verification introduces additional exploitation opportunities whereas VLM-based semantic verification can reduce reward hacking. Overall, multimodal reward hacking is a systematic consequence of optimizing imperfect reward signals, highlighting the need for rewards and verifiers that remain reliable under optimization pressure.</abstract>
<identifier type="citekey">yao-etal-2026-multimodal</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.980/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>20718</start>
<end>20733</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Multimodal Reward Hacking in Reinforcement Learning
%A Yao, Jiayu
%A Wang, Yiwei
%A Zhang, Anmeng
%A Sun, Zhe
%A Wang, Songsong
%A Mei, Lingrui
%A Ge, Yuyao
%A Liu, Shenghua
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F yao-etal-2026-multimodal
%X Reinforcement learning is increasingly used to align multimodal large language models (MLLMs). However, higher proxy rewards do not necessarily lead to better task performance, especially when visual evidence is difficult to verify with text-only or weakly grounded reward signals. We systematically study reward hacking in MLLM reinforcement learning by varying reward design, data ambiguity, model scale, and RL algorithm. To separate RL-induced failures from pre-existing weaknesses, we introduce the Newly Rewarded Failure Rate (NRFR), which measures the hacking rate among samples where RL achieves higher proxy reward than the SFT baseline. Our analysis yields three main findings. First, RL can actively create new failures rather than merely amplify existing ones, and the effect is especially pronounced near ambiguous decision boundaries. Second, robustness depends jointly on model scale, reward design, and RL algorithm, and scaling mitigates hacking but cannot replace reliable rewards. Third, incorporating visual evidence helps only when verification is reliable, since keyword-based verification introduces additional exploitation opportunities whereas VLM-based semantic verification can reduce reward hacking. Overall, multimodal reward hacking is a systematic consequence of optimizing imperfect reward signals, highlighting the need for rewards and verifiers that remain reliable under optimization pressure.
%U https://aclanthology.org/2026.findings-emnlp.980/
%P 20718-20733
Markdown (Informal)
[Multimodal Reward Hacking in Reinforcement Learning](https://aclanthology.org/2026.findings-emnlp.980/) (Yao et al., Findings 2026)
ACL
- Jiayu Yao, Yiwei Wang, Anmeng Zhang, Zhe Sun, Songsong Wang, Lingrui Mei, Yuyao Ge, and Shenghua Liu. 2026. Multimodal Reward Hacking in Reinforcement Learning. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 20718–20733, Budapest, Hungary. Association for Computational Linguistics.