@inproceedings{clervilsson-guo-2026-evaluating,
title = "Evaluating Cultural Awareness of {LLM}s for {H}aitian {C}reole",
author = "Clervilsson, Christelle and
Guo, Yanzhu",
editor = "Xu, Shanshan and
T.Y.S.S., Santosh and
Kurniawan, Kemal and
Peng, Siyao and
Wallbridge, Sarenne and
Koto, Fajri and
Dziri, Nouha and
Plank, Barbara",
booktitle = "Proceedings of the First Workshop on Pluralistic {AI} {NLP}: Diversity-aware, Sociotechnical, Responsible Alignment ({PANDORA} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.pandora-1.21/",
pages = "373--387",
ISBN = "979-8-89176-484-2",
abstract = "Large language models (LLMs) exhibit substantial performance disparities between high- and low-resource languages. Beyond lower task performance, they often fail to capture the cultural norms and values of underrepresented communities. In this work, we present the first systematic evaluation of cultural awareness in LLMs for Haitian Creole, a language spoken by millions but severely underrepresented in digital resources. We assess cultural awareness along four complementary dimensions{---}specificity, bias, diversity, and variation{---}using a benchmark of culturally salient prompts curated by native speakers in a text infilling setting. Our results reveal a clear gap between cultural awareness in Haitian Creole and higher-resource French, with Haitian performance being more uneven across domains and more affected by French linguistic interference. Story generation further reveals recurring portrayals of Haitian characters through hardship and resilience, showing that even positive characterizations can encode stereotypical narratives. We will make our code, benchmark, and evaluation framework publicly available upon acceptance."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="clervilsson-guo-2026-evaluating">
<titleInfo>
<title>Evaluating Cultural Awareness of LLMs for Haitian Creole</title>
</titleInfo>
<name type="personal">
<namePart type="given">Christelle</namePart>
<namePart type="family">Clervilsson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yanzhu</namePart>
<namePart type="family">Guo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shanshan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Santosh</namePart>
<namePart type="family">T.Y.S.S.</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kemal</namePart>
<namePart type="family">Kurniawan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siyao</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sarenne</namePart>
<namePart type="family">Wallbridge</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fajri</namePart>
<namePart type="family">Koto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nouha</namePart>
<namePart type="family">Dziri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-484-2</identifier>
</relatedItem>
<abstract>Large language models (LLMs) exhibit substantial performance disparities between high- and low-resource languages. Beyond lower task performance, they often fail to capture the cultural norms and values of underrepresented communities. In this work, we present the first systematic evaluation of cultural awareness in LLMs for Haitian Creole, a language spoken by millions but severely underrepresented in digital resources. We assess cultural awareness along four complementary dimensions—specificity, bias, diversity, and variation—using a benchmark of culturally salient prompts curated by native speakers in a text infilling setting. Our results reveal a clear gap between cultural awareness in Haitian Creole and higher-resource French, with Haitian performance being more uneven across domains and more affected by French linguistic interference. Story generation further reveals recurring portrayals of Haitian characters through hardship and resilience, showing that even positive characterizations can encode stereotypical narratives. We will make our code, benchmark, and evaluation framework publicly available upon acceptance.</abstract>
<identifier type="citekey">clervilsson-guo-2026-evaluating</identifier>
<location>
<url>https://aclanthology.org/2026.pandora-1.21/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>373</start>
<end>387</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating Cultural Awareness of LLMs for Haitian Creole
%A Clervilsson, Christelle
%A Guo, Yanzhu
%Y Xu, Shanshan
%Y T.Y.S.S., Santosh
%Y Kurniawan, Kemal
%Y Peng, Siyao
%Y Wallbridge, Sarenne
%Y Koto, Fajri
%Y Dziri, Nouha
%Y Plank, Barbara
%S Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-484-2
%F clervilsson-guo-2026-evaluating
%X Large language models (LLMs) exhibit substantial performance disparities between high- and low-resource languages. Beyond lower task performance, they often fail to capture the cultural norms and values of underrepresented communities. In this work, we present the first systematic evaluation of cultural awareness in LLMs for Haitian Creole, a language spoken by millions but severely underrepresented in digital resources. We assess cultural awareness along four complementary dimensions—specificity, bias, diversity, and variation—using a benchmark of culturally salient prompts curated by native speakers in a text infilling setting. Our results reveal a clear gap between cultural awareness in Haitian Creole and higher-resource French, with Haitian performance being more uneven across domains and more affected by French linguistic interference. Story generation further reveals recurring portrayals of Haitian characters through hardship and resilience, showing that even positive characterizations can encode stereotypical narratives. We will make our code, benchmark, and evaluation framework publicly available upon acceptance.
%U https://aclanthology.org/2026.pandora-1.21/
%P 373-387
Markdown (Informal)
[Evaluating Cultural Awareness of LLMs for Haitian Creole](https://aclanthology.org/2026.pandora-1.21/) (Clervilsson & Guo, PANDORA 2026)
ACL
- Christelle Clervilsson and Yanzhu Guo. 2026. Evaluating Cultural Awareness of LLMs for Haitian Creole. In Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026), pages 373–387, Budapest, Hungary. Association for Computational Linguistics.