@inproceedings{lee-etal-2026-kmmmu,
title = "{KMMMU}: Evaluation of Massive Multi-discipline Multimodal Understanding in {K}orean Language and Context",
author = "Lee, Nahyun and
Son, Guijin and
Ko, Hyunwoo and
Kim, Chanyoung and
An, JunYoung and
Han, Kyubeen and
Yu, Youngjae and
Kwak, Il-Youp",
editor = "Adelani, David Ifeoluwa and
Alabi, Jesujoba and
Arnett, Catherine and
Chang, Tyler and
Downey, C.M. and
Goldman, Omer and
Li, Senyu and
Miranda, Lester James Validad and
Qi, Jirui and
Stap, David and
Winata, Genta Indra and
Zhang, Xinyu Crystina",
booktitle = "Proceedings of the 6th Workshop on Multilingual Representation Learning ({MRL} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mrl-1.47/",
pages = "609--634",
ISBN = "979-8-89176-469-9",
abstract = "Multimodal evaluation is strengthened when benchmarks reflect the languages, institutions, and disciplinary conventions that shape expert visual reasoning. We introduce KMMMU, a native Korean benchmark built from Korean-native examination materials. KMMMU contains 3,466 questions spanning nine disciplines and nine visual modality categories. For targeted evaluation on challenging questions and Korean-specific knowledge, we additionally provide KMMMU-Adv and KMMMU-Local, respectively. Experiments show that the strongest evaluated open-weight model reaches 42.05{\%} on the KMMMU, while the best evaluated proprietary model achieves 52.42{\%} on the KMMMU-Adv. Performance varies substantially across disciplines and answer formats, with Korean-specific questions showing gaps of up to 13.43{\%}. Error analysis suggests that inspected failures often involve convention-to-label mapping, few-shot symbolic induction, localized knowledge recall, and domain-specific standards, not reasoning depth alone. KMMMU provides a testbed for native Korean expert-level multimodal understanding beyond English-centric or translation-based evaluation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lee-etal-2026-kmmmu">
<titleInfo>
<title>KMMMU: Evaluation of Massive Multi-discipline Multimodal Understanding in Korean Language and Context</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nahyun</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Guijin</namePart>
<namePart type="family">Son</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hyunwoo</namePart>
<namePart type="family">Ko</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chanyoung</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">JunYoung</namePart>
<namePart type="family">An</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kyubeen</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Youngjae</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Il-Youp</namePart>
<namePart type="family">Kwak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Ifeoluwa</namePart>
<namePart type="family">Adelani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesujoba</namePart>
<namePart type="family">Alabi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Catherine</namePart>
<namePart type="family">Arnett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">C</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Downey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omer</namePart>
<namePart type="family">Goldman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Senyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lester</namePart>
<namePart type="given">James</namePart>
<namePart type="given">Validad</namePart>
<namePart type="family">Miranda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jirui</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Stap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Genta</namePart>
<namePart type="given">Indra</namePart>
<namePart type="family">Winata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="given">Crystina</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-469-9</identifier>
</relatedItem>
<abstract>Multimodal evaluation is strengthened when benchmarks reflect the languages, institutions, and disciplinary conventions that shape expert visual reasoning. We introduce KMMMU, a native Korean benchmark built from Korean-native examination materials. KMMMU contains 3,466 questions spanning nine disciplines and nine visual modality categories. For targeted evaluation on challenging questions and Korean-specific knowledge, we additionally provide KMMMU-Adv and KMMMU-Local, respectively. Experiments show that the strongest evaluated open-weight model reaches 42.05% on the KMMMU, while the best evaluated proprietary model achieves 52.42% on the KMMMU-Adv. Performance varies substantially across disciplines and answer formats, with Korean-specific questions showing gaps of up to 13.43%. Error analysis suggests that inspected failures often involve convention-to-label mapping, few-shot symbolic induction, localized knowledge recall, and domain-specific standards, not reasoning depth alone. KMMMU provides a testbed for native Korean expert-level multimodal understanding beyond English-centric or translation-based evaluation.</abstract>
<identifier type="citekey">lee-etal-2026-kmmmu</identifier>
<location>
<url>https://aclanthology.org/2026.mrl-1.47/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>609</start>
<end>634</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T KMMMU: Evaluation of Massive Multi-discipline Multimodal Understanding in Korean Language and Context
%A Lee, Nahyun
%A Son, Guijin
%A Ko, Hyunwoo
%A Kim, Chanyoung
%A An, JunYoung
%A Han, Kyubeen
%A Yu, Youngjae
%A Kwak, Il-Youp
%Y Adelani, David Ifeoluwa
%Y Alabi, Jesujoba
%Y Arnett, Catherine
%Y Chang, Tyler
%Y Downey, C. M.
%Y Goldman, Omer
%Y Li, Senyu
%Y Miranda, Lester James Validad
%Y Qi, Jirui
%Y Stap, David
%Y Winata, Genta Indra
%Y Zhang, Xinyu Crystina
%S Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-469-9
%F lee-etal-2026-kmmmu
%X Multimodal evaluation is strengthened when benchmarks reflect the languages, institutions, and disciplinary conventions that shape expert visual reasoning. We introduce KMMMU, a native Korean benchmark built from Korean-native examination materials. KMMMU contains 3,466 questions spanning nine disciplines and nine visual modality categories. For targeted evaluation on challenging questions and Korean-specific knowledge, we additionally provide KMMMU-Adv and KMMMU-Local, respectively. Experiments show that the strongest evaluated open-weight model reaches 42.05% on the KMMMU, while the best evaluated proprietary model achieves 52.42% on the KMMMU-Adv. Performance varies substantially across disciplines and answer formats, with Korean-specific questions showing gaps of up to 13.43%. Error analysis suggests that inspected failures often involve convention-to-label mapping, few-shot symbolic induction, localized knowledge recall, and domain-specific standards, not reasoning depth alone. KMMMU provides a testbed for native Korean expert-level multimodal understanding beyond English-centric or translation-based evaluation.
%U https://aclanthology.org/2026.mrl-1.47/
%P 609-634
Markdown (Informal)
[KMMMU: Evaluation of Massive Multi-discipline Multimodal Understanding in Korean Language and Context](https://aclanthology.org/2026.mrl-1.47/) (Lee et al., MRL 2026)
ACL
- Nahyun Lee, Guijin Son, Hyunwoo Ko, Chanyoung Kim, JunYoung An, Kyubeen Han, Youngjae Yu, and Il-Youp Kwak. 2026. KMMMU: Evaluation of Massive Multi-discipline Multimodal Understanding in Korean Language and Context. In Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026), pages 609–634, Budapest, Hungary. Association for Computational Linguistics.