@inproceedings{thamatam-etal-2026-method,
title = "Method Ranking Inverts Across Targets: A Matched-Grid Evaluation of Jailbreak Attacks on Open-Weight Vision-Language Models",
author = "Thamatam, Venkat Aniruddh and
Majhi, Vasudev and
Nagda, Aaryan and
Challa, Jagat Sesh and
Mandal, Murari and
Sinha, Yash and
Kumar, Dhruv",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.50/",
pages = "1043--1063",
ISBN = "979-8-89176-439-2",
abstract = "Jailbreak attacks are usually compared by their attack success rate (ASR), with a higher ASR read as a stronger attack. But an ASR mixes an attack{'}s general strength with its fit to the particular model it was measured on, and it also depends on the judge that scored it. We separate these factors with a matched evaluation grid: five attacks on eight open-weight vision{--}language models, 200 HarmBench behaviours per cell, every turn scored by three judges (Qwen2.5-7B, Qwen2.5-72B and Llama-Guard-3), and each ASR decomposed into general strength, target susceptibility and target-specific fit. Attack rankings invert across targets: FigStep leads PAIR on InternVL2.5-8B (13.0{\%} vs. 3.5{\%}, paired $p{=}4{\times}10^{-10}$) and trails it on Qwen2.5-VL-7B, in the same direction under all three judges. Across the grid, the target-specific component is statistically significant under every judge ($p{<}10^{-12}$). The judge matters as well: agreement between judges is low (Cohen{'}s $\kappa$ between 0.03 and 0.29), and a larger judge with the same rubric changes which attack ranks first on three of eight targets. A higher ASR therefore shows that an attack succeeded more often on a given model under a given judge, but not whether its advantage is general or specific to that model. We release the protocol and ${\sim}20{,}400$ trajectories with per-turn scores under all three judges"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="thamatam-etal-2026-method">
<titleInfo>
<title>Method Ranking Inverts Across Targets: A Matched-Grid Evaluation of Jailbreak Attacks on Open-Weight Vision-Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Venkat</namePart>
<namePart type="given">Aniruddh</namePart>
<namePart type="family">Thamatam</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vasudev</namePart>
<namePart type="family">Majhi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaryan</namePart>
<namePart type="family">Nagda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jagat</namePart>
<namePart type="given">Sesh</namePart>
<namePart type="family">Challa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Murari</namePart>
<namePart type="family">Mandal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yash</namePart>
<namePart type="family">Sinha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dhruv</namePart>
<namePart type="family">Kumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Jailbreak attacks are usually compared by their attack success rate (ASR), with a higher ASR read as a stronger attack. But an ASR mixes an attack’s general strength with its fit to the particular model it was measured on, and it also depends on the judge that scored it. We separate these factors with a matched evaluation grid: five attacks on eight open-weight vision–language models, 200 HarmBench behaviours per cell, every turn scored by three judges (Qwen2.5-7B, Qwen2.5-72B and Llama-Guard-3), and each ASR decomposed into general strength, target susceptibility and target-specific fit. Attack rankings invert across targets: FigStep leads PAIR on InternVL2.5-8B (13.0% vs. 3.5%, paired p=4\times10⁻10) and trails it on Qwen2.5-VL-7B, in the same direction under all three judges. Across the grid, the target-specific component is statistically significant under every judge (p<10⁻12). The judge matters as well: agreement between judges is low (Cohen’s ąppa between 0.03 and 0.29), and a larger judge with the same rubric changes which attack ranks first on three of eight targets. A higher ASR therefore shows that an attack succeeded more often on a given model under a given judge, but not whether its advantage is general or specific to that model. We release the protocol and \sim20,400 trajectories with per-turn scores under all three judges</abstract>
<identifier type="citekey">thamatam-etal-2026-method</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.50/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>1043</start>
<end>1063</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Method Ranking Inverts Across Targets: A Matched-Grid Evaluation of Jailbreak Attacks on Open-Weight Vision-Language Models
%A Thamatam, Venkat Aniruddh
%A Majhi, Vasudev
%A Nagda, Aaryan
%A Challa, Jagat Sesh
%A Mandal, Murari
%A Sinha, Yash
%A Kumar, Dhruv
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F thamatam-etal-2026-method
%X Jailbreak attacks are usually compared by their attack success rate (ASR), with a higher ASR read as a stronger attack. But an ASR mixes an attack’s general strength with its fit to the particular model it was measured on, and it also depends on the judge that scored it. We separate these factors with a matched evaluation grid: five attacks on eight open-weight vision–language models, 200 HarmBench behaviours per cell, every turn scored by three judges (Qwen2.5-7B, Qwen2.5-72B and Llama-Guard-3), and each ASR decomposed into general strength, target susceptibility and target-specific fit. Attack rankings invert across targets: FigStep leads PAIR on InternVL2.5-8B (13.0% vs. 3.5%, paired p=4\times10⁻10) and trails it on Qwen2.5-VL-7B, in the same direction under all three judges. Across the grid, the target-specific component is statistically significant under every judge (p<10⁻12). The judge matters as well: agreement between judges is low (Cohen’s ąppa between 0.03 and 0.29), and a larger judge with the same rubric changes which attack ranks first on three of eight targets. A higher ASR therefore shows that an attack succeeded more often on a given model under a given judge, but not whether its advantage is general or specific to that model. We release the protocol and \sim20,400 trajectories with per-turn scores under all three judges
%U https://aclanthology.org/2026.aacl-long.50/
%P 1043-1063
Markdown (Informal)
[Method Ranking Inverts Across Targets: A Matched-Grid Evaluation of Jailbreak Attacks on Open-Weight Vision-Language Models](https://aclanthology.org/2026.aacl-long.50/) (Thamatam et al., AACL-IJCNLP 2026)
ACL