@inproceedings{holtermann-etal-2026-cultural,
title = "What is ``Cultural Representation'' in {AI} Benchmarks? A Multidimensional Evaluation Framework",
author = "Holtermann, Carolin and
Birkmann, Josefine and
Lauscher, Anne",
editor = "Xu, Shanshan and
T.Y.S.S., Santosh and
Kurniawan, Kemal and
Peng, Siyao and
Wallbridge, Sarenne and
Koto, Fajri and
Dziri, Nouha and
Plank, Barbara",
booktitle = "Proceedings of the First Workshop on Pluralistic {AI} {NLP}: Diversity-aware, Sociotechnical, Responsible Alignment ({PANDORA} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.pandora-1.3/",
pages = "69--87",
ISBN = "979-8-89176-484-2",
abstract = "Cultural representation is a prerequisite for the comprehensive evaluation of cultural competence in AI systems. Yet there is no shared, structured way to assess what cultural representation in AI benchmarks involves or how it can be measured. We address this gap by introducing a six-dimensional framework that assesses cultural representation through a structured, dimension-specific scoring instrument, enabling systematic comparison across benchmarks. Applying our framework to 50 benchmarks explicitly targeting culture, we find that none achieves broad coverage across all dimensions. While procedural dimensions such as transparency and bias awareness are comparatively well addressed, benchmarks consistently score lower in linguistic and geographic representation, thematic breadth, and the conceptual grounding of culture. Strikingly, only 32{\%} of benchmarks explicitly define culture, and annotator demographic information is absent in 82{\%} of cases. By making these properties measurable and directly comparable, our framework provides a practical foundation for designing and evaluating benchmarks that more validly assess cultural competence in AI systems."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="holtermann-etal-2026-cultural">
<titleInfo>
<title>What is “Cultural Representation” in AI Benchmarks? A Multidimensional Evaluation Framework</title>
</titleInfo>
<name type="personal">
<namePart type="given">Carolin</namePart>
<namePart type="family">Holtermann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Josefine</namePart>
<namePart type="family">Birkmann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anne</namePart>
<namePart type="family">Lauscher</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shanshan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Santosh</namePart>
<namePart type="family">T.Y.S.S.</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kemal</namePart>
<namePart type="family">Kurniawan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siyao</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sarenne</namePart>
<namePart type="family">Wallbridge</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fajri</namePart>
<namePart type="family">Koto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nouha</namePart>
<namePart type="family">Dziri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-484-2</identifier>
</relatedItem>
<abstract>Cultural representation is a prerequisite for the comprehensive evaluation of cultural competence in AI systems. Yet there is no shared, structured way to assess what cultural representation in AI benchmarks involves or how it can be measured. We address this gap by introducing a six-dimensional framework that assesses cultural representation through a structured, dimension-specific scoring instrument, enabling systematic comparison across benchmarks. Applying our framework to 50 benchmarks explicitly targeting culture, we find that none achieves broad coverage across all dimensions. While procedural dimensions such as transparency and bias awareness are comparatively well addressed, benchmarks consistently score lower in linguistic and geographic representation, thematic breadth, and the conceptual grounding of culture. Strikingly, only 32% of benchmarks explicitly define culture, and annotator demographic information is absent in 82% of cases. By making these properties measurable and directly comparable, our framework provides a practical foundation for designing and evaluating benchmarks that more validly assess cultural competence in AI systems.</abstract>
<identifier type="citekey">holtermann-etal-2026-cultural</identifier>
<location>
<url>https://aclanthology.org/2026.pandora-1.3/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>69</start>
<end>87</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What is “Cultural Representation” in AI Benchmarks? A Multidimensional Evaluation Framework
%A Holtermann, Carolin
%A Birkmann, Josefine
%A Lauscher, Anne
%Y Xu, Shanshan
%Y T.Y.S.S., Santosh
%Y Kurniawan, Kemal
%Y Peng, Siyao
%Y Wallbridge, Sarenne
%Y Koto, Fajri
%Y Dziri, Nouha
%Y Plank, Barbara
%S Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-484-2
%F holtermann-etal-2026-cultural
%X Cultural representation is a prerequisite for the comprehensive evaluation of cultural competence in AI systems. Yet there is no shared, structured way to assess what cultural representation in AI benchmarks involves or how it can be measured. We address this gap by introducing a six-dimensional framework that assesses cultural representation through a structured, dimension-specific scoring instrument, enabling systematic comparison across benchmarks. Applying our framework to 50 benchmarks explicitly targeting culture, we find that none achieves broad coverage across all dimensions. While procedural dimensions such as transparency and bias awareness are comparatively well addressed, benchmarks consistently score lower in linguistic and geographic representation, thematic breadth, and the conceptual grounding of culture. Strikingly, only 32% of benchmarks explicitly define culture, and annotator demographic information is absent in 82% of cases. By making these properties measurable and directly comparable, our framework provides a practical foundation for designing and evaluating benchmarks that more validly assess cultural competence in AI systems.
%U https://aclanthology.org/2026.pandora-1.3/
%P 69-87
Markdown (Informal)
[What is "Cultural Representation” in AI Benchmarks? A Multidimensional Evaluation Framework](https://aclanthology.org/2026.pandora-1.3/) (Holtermann et al., PANDORA 2026)
ACL