@inproceedings{lee-etal-2026-jury,
title = "Jury Duty: Calibration and Orientation Failures in {MLLM}-as-a-Judge Under Cultural Ambiguity",
author = "Lee, Daniel and
Sharma, Harshit and
Park, Eunkyu and
Venkit, Pranav Narayanan and
Kim, Jeonghwan and
Chia, Kah Mun and
Vlachos, Andreas and
Joty, Shafiq",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2467/",
pages = "56015--56032",
ISBN = "979-8-89176-459-0",
abstract = "MLLM-as-a-Judge is conventionally validated by agreement with human annotations, but agreement against a culturally heterogeneous pool is pool-dependent: it silently conditions on whose judgments the aggregate represents. We introduce VOIR DIRE, a multimodal benchmark of 626 culturally paired image{--}prompt items spanning U.S. and mainland Chinese contexts across food, fashion, and architecture, with annotator pools that are within-pool reliable ($\alpha = 0.86 / 0.74$) but cross-pool divergent on evaluation (Q1 $r = -0.12$). Across six MLLM judges, the bias decomposes into two failures: a positivity-floor calibration failure (compressed scale use) and an orientation failure (default to one cultural norm). On this corpus, contested items are sampled so the two pools split, and because models default to high ratings, the floor mechanically sides with the more-permissive Chinese reading. Persona prompting partially recovers calibration, but an orientation residual survives{---}evidence the tilt is not reducible to scale compression. Reference-pool in-context demonstrations deepen the orientation residual and inflate the high end rather than restoring use of the low end. Model origin adds a small additive tilt ($\sim 0.10$ MAE) that is approximately invariant under demonstration. We recommend reporting alignment against each reference pool separately and treating cross-pool divergence as a judge property."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lee-etal-2026-jury">
<titleInfo>
<title>Jury Duty: Calibration and Orientation Failures in MLLM-as-a-Judge Under Cultural Ambiguity</title>
</titleInfo>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Harshit</namePart>
<namePart type="family">Sharma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Eunkyu</namePart>
<namePart type="family">Park</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pranav</namePart>
<namePart type="given">Narayanan</namePart>
<namePart type="family">Venkit</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jeonghwan</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kah</namePart>
<namePart type="given">Mun</namePart>
<namePart type="family">Chia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andreas</namePart>
<namePart type="family">Vlachos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shafiq</namePart>
<namePart type="family">Joty</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>MLLM-as-a-Judge is conventionally validated by agreement with human annotations, but agreement against a culturally heterogeneous pool is pool-dependent: it silently conditions on whose judgments the aggregate represents. We introduce VOIR DIRE, a multimodal benchmark of 626 culturally paired image–prompt items spanning U.S. and mainland Chinese contexts across food, fashion, and architecture, with annotator pools that are within-pool reliable (α = 0.86 / 0.74) but cross-pool divergent on evaluation (Q1 r = -0.12). Across six MLLM judges, the bias decomposes into two failures: a positivity-floor calibration failure (compressed scale use) and an orientation failure (default to one cultural norm). On this corpus, contested items are sampled so the two pools split, and because models default to high ratings, the floor mechanically sides with the more-permissive Chinese reading. Persona prompting partially recovers calibration, but an orientation residual survives—evidence the tilt is not reducible to scale compression. Reference-pool in-context demonstrations deepen the orientation residual and inflate the high end rather than restoring use of the low end. Model origin adds a small additive tilt (\sim 0.10 MAE) that is approximately invariant under demonstration. We recommend reporting alignment against each reference pool separately and treating cross-pool divergence as a judge property.</abstract>
<identifier type="citekey">lee-etal-2026-jury</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2467/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>56015</start>
<end>56032</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Jury Duty: Calibration and Orientation Failures in MLLM-as-a-Judge Under Cultural Ambiguity
%A Lee, Daniel
%A Sharma, Harshit
%A Park, Eunkyu
%A Venkit, Pranav Narayanan
%A Kim, Jeonghwan
%A Chia, Kah Mun
%A Vlachos, Andreas
%A Joty, Shafiq
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F lee-etal-2026-jury
%X MLLM-as-a-Judge is conventionally validated by agreement with human annotations, but agreement against a culturally heterogeneous pool is pool-dependent: it silently conditions on whose judgments the aggregate represents. We introduce VOIR DIRE, a multimodal benchmark of 626 culturally paired image–prompt items spanning U.S. and mainland Chinese contexts across food, fashion, and architecture, with annotator pools that are within-pool reliable (α = 0.86 / 0.74) but cross-pool divergent on evaluation (Q1 r = -0.12). Across six MLLM judges, the bias decomposes into two failures: a positivity-floor calibration failure (compressed scale use) and an orientation failure (default to one cultural norm). On this corpus, contested items are sampled so the two pools split, and because models default to high ratings, the floor mechanically sides with the more-permissive Chinese reading. Persona prompting partially recovers calibration, but an orientation residual survives—evidence the tilt is not reducible to scale compression. Reference-pool in-context demonstrations deepen the orientation residual and inflate the high end rather than restoring use of the low end. Model origin adds a small additive tilt (\sim 0.10 MAE) that is approximately invariant under demonstration. We recommend reporting alignment against each reference pool separately and treating cross-pool divergence as a judge property.
%U https://aclanthology.org/2026.emnlp-main.2467/
%P 56015-56032
Markdown (Informal)
[Jury Duty: Calibration and Orientation Failures in MLLM-as-a-Judge Under Cultural Ambiguity](https://aclanthology.org/2026.emnlp-main.2467/) (Lee et al., EMNLP 2026)
ACL
- Daniel Lee, Harshit Sharma, Eunkyu Park, Pranav Narayanan Venkit, Jeonghwan Kim, Kah Mun Chia, Andreas Vlachos, and Shafiq Joty. 2026. Jury Duty: Calibration and Orientation Failures in MLLM-as-a-Judge Under Cultural Ambiguity. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 56015–56032, Budapest, Hungary. Association for Computational Linguistics.