@inproceedings{nyachhyon-etal-2026-nepplural,
title = "{N}ep{P}lural: A Human-Anchored Benchmark for Persona Annotation in {N}epali Migration Discourse",
author = "Nyachhyon, Jinu and
Khatiwada, Anish and
Timilsina, Bimal",
editor = "Xiong, Deyi and
Branco, Ant{\'o}nio and
Zhang, Hongming and
Dong, Yue and
Wang, Benyou and
Zhang, Wenxuan and
Zhou, Li and
Zheng, Jingting",
booktitle = "Proceedings of the First Workshop on Pluralistic Value Alignment of {LLM}s",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.plurvallm-1.15/",
pages = "197--210",
ISBN = "979-8-89176-453-8",
abstract = "Pluralistic alignment asks that language models faithfully represent a society{'}s diverse, competing values rather than collapse them into one dominant viewpoint, yet flagship pluralistic-alignment datasets remain heavily skewed toward Western respondents. We introduce NepPlural, a human-anchored benchmark for modelling competing socio-cultural personas in Nepali public discourse on youth migration and brain drain. We curate 2,148 code-mixed (Devanagari, Romanized Nepali, English) YouTube comments, annotate them along a four-axis persona taxonomy (Intent, Primary Driver, Value Orientation, Affect) plus an explicit No-Persona class, and build a 300-comment gold set labelled independently by three native Nepali speakers (mean Fleiss' $\kappa \approx 0.55$). Using this gold set to audit automatic annotation, we find that LLM annotation reliability is driven by protocol. Labels collected in batches through chat interfaces reach only near-chance inter-model agreement ($\kappa \approx 0.17$). Annotating one comment at a time raises agreement above the native-speaker level ($\kappa \approx 0.72$), but consistency is not validity: under a forced-choice schema, models agree on personas for the 39{\%} of comments native speakers judge to express none, keeping macro-F1 against human gold near 0.50. Allowing abstention resolves this: models recover No-Persona comments with F1 of 0.79 to 0.87 and, against leave-one-annotator-out references, match or exceed individual native speakers (macro-F1 0.75 to 0.86 vs. 0.75). Encoders fine-tuned on the batched labels reach only 0.34. Finally, a human-judged steerability probe finds that three frontier models adopt single persona axes with 59 to 77{\%} fidelity but compose full personas only 33{\%} of the time, with no congruence gap detected at this scale between Western-congruent and Nepali-collectivist personas. We release the data, code, prompts, and all model outputs."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="nyachhyon-etal-2026-nepplural">
<titleInfo>
<title>NepPlural: A Human-Anchored Benchmark for Persona Annotation in Nepali Migration Discourse</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jinu</namePart>
<namePart type="family">Nyachhyon</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anish</namePart>
<namePart type="family">Khatiwada</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bimal</namePart>
<namePart type="family">Timilsina</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Pluralistic Value Alignment of LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Deyi</namePart>
<namePart type="family">Xiong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">António</namePart>
<namePart type="family">Branco</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hongming</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Dong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Benyou</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenxuan</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Li</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingting</namePart>
<namePart type="family">Zheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-453-8</identifier>
</relatedItem>
<abstract>Pluralistic alignment asks that language models faithfully represent a society’s diverse, competing values rather than collapse them into one dominant viewpoint, yet flagship pluralistic-alignment datasets remain heavily skewed toward Western respondents. We introduce NepPlural, a human-anchored benchmark for modelling competing socio-cultural personas in Nepali public discourse on youth migration and brain drain. We curate 2,148 code-mixed (Devanagari, Romanized Nepali, English) YouTube comments, annotate them along a four-axis persona taxonomy (Intent, Primary Driver, Value Orientation, Affect) plus an explicit No-Persona class, and build a 300-comment gold set labelled independently by three native Nepali speakers (mean Fleiss’ ąppa \approx 0.55). Using this gold set to audit automatic annotation, we find that LLM annotation reliability is driven by protocol. Labels collected in batches through chat interfaces reach only near-chance inter-model agreement (ąppa \approx 0.17). Annotating one comment at a time raises agreement above the native-speaker level (ąppa \approx 0.72), but consistency is not validity: under a forced-choice schema, models agree on personas for the 39% of comments native speakers judge to express none, keeping macro-F1 against human gold near 0.50. Allowing abstention resolves this: models recover No-Persona comments with F1 of 0.79 to 0.87 and, against leave-one-annotator-out references, match or exceed individual native speakers (macro-F1 0.75 to 0.86 vs. 0.75). Encoders fine-tuned on the batched labels reach only 0.34. Finally, a human-judged steerability probe finds that three frontier models adopt single persona axes with 59 to 77% fidelity but compose full personas only 33% of the time, with no congruence gap detected at this scale between Western-congruent and Nepali-collectivist personas. We release the data, code, prompts, and all model outputs.</abstract>
<identifier type="citekey">nyachhyon-etal-2026-nepplural</identifier>
<location>
<url>https://aclanthology.org/2026.plurvallm-1.15/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>197</start>
<end>210</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T NepPlural: A Human-Anchored Benchmark for Persona Annotation in Nepali Migration Discourse
%A Nyachhyon, Jinu
%A Khatiwada, Anish
%A Timilsina, Bimal
%Y Xiong, Deyi
%Y Branco, António
%Y Zhang, Hongming
%Y Dong, Yue
%Y Wang, Benyou
%Y Zhang, Wenxuan
%Y Zhou, Li
%Y Zheng, Jingting
%S Proceedings of the First Workshop on Pluralistic Value Alignment of LLMs
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-453-8
%F nyachhyon-etal-2026-nepplural
%X Pluralistic alignment asks that language models faithfully represent a society’s diverse, competing values rather than collapse them into one dominant viewpoint, yet flagship pluralistic-alignment datasets remain heavily skewed toward Western respondents. We introduce NepPlural, a human-anchored benchmark for modelling competing socio-cultural personas in Nepali public discourse on youth migration and brain drain. We curate 2,148 code-mixed (Devanagari, Romanized Nepali, English) YouTube comments, annotate them along a four-axis persona taxonomy (Intent, Primary Driver, Value Orientation, Affect) plus an explicit No-Persona class, and build a 300-comment gold set labelled independently by three native Nepali speakers (mean Fleiss’ ąppa \approx 0.55). Using this gold set to audit automatic annotation, we find that LLM annotation reliability is driven by protocol. Labels collected in batches through chat interfaces reach only near-chance inter-model agreement (ąppa \approx 0.17). Annotating one comment at a time raises agreement above the native-speaker level (ąppa \approx 0.72), but consistency is not validity: under a forced-choice schema, models agree on personas for the 39% of comments native speakers judge to express none, keeping macro-F1 against human gold near 0.50. Allowing abstention resolves this: models recover No-Persona comments with F1 of 0.79 to 0.87 and, against leave-one-annotator-out references, match or exceed individual native speakers (macro-F1 0.75 to 0.86 vs. 0.75). Encoders fine-tuned on the batched labels reach only 0.34. Finally, a human-judged steerability probe finds that three frontier models adopt single persona axes with 59 to 77% fidelity but compose full personas only 33% of the time, with no congruence gap detected at this scale between Western-congruent and Nepali-collectivist personas. We release the data, code, prompts, and all model outputs.
%U https://aclanthology.org/2026.plurvallm-1.15/
%P 197-210
Markdown (Informal)
[NepPlural: A Human-Anchored Benchmark for Persona Annotation in Nepali Migration Discourse](https://aclanthology.org/2026.plurvallm-1.15/) (Nyachhyon et al., PlurVA-LLM 2026)
ACL