@inproceedings{kamruzzaman-etal-2026-demographic,
title = "Demographic Prompting at Scale: When More Attributes Hurt {LLM}{--}Human Agreement",
author = "Kamruzzaman, Mahammed and
Das, Shrabon Kumar and
Kim, Gene Louis",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.60/",
pages = "1373--1399",
ISBN = "979-8-89176-459-0",
abstract = "We investigate how annotator demographic attributes, supplied as prompt cues, shape the alignment between large language model (LLM) predictions and human annotations across five tasks. Using five open-source LLMs, we systematically vary the number and composition of demographic components in the prompt, spanning every combination from single-attribute through full-attribute configurations. Our experiments reveal three principal findings. First, demographic prompting is highly model- and task-dependent. The baseline is best in 12 of 25 model{--}task pairs. Among the remaining 13, 11 peak with 1{--}3 attributes, while two require higher-order combinations. The full-attribute prompt is never the best configuration. Second, the overall magnitude of demographic influence on human annotations does not predict which attributes improve LLM alignment; instead, both the learnability and the directional coherence of each attribute{'}s annotation signal need to be considered jointly. Third, neuron probing reveals that specialized activation correlates with alignment gains only under coherent annotation signals, and that activation volume alone does not imply steerability. Together, these results demonstrate that demographic prompting is not a monolithic intervention: its utility is highly context-dependent, shaped by attribute signal quality, task characteristics, and model architecture."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kamruzzaman-etal-2026-demographic">
<titleInfo>
<title>Demographic Prompting at Scale: When More Attributes Hurt LLM–Human Agreement</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mahammed</namePart>
<namePart type="family">Kamruzzaman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shrabon</namePart>
<namePart type="given">Kumar</namePart>
<namePart type="family">Das</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gene</namePart>
<namePart type="given">Louis</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>We investigate how annotator demographic attributes, supplied as prompt cues, shape the alignment between large language model (LLM) predictions and human annotations across five tasks. Using five open-source LLMs, we systematically vary the number and composition of demographic components in the prompt, spanning every combination from single-attribute through full-attribute configurations. Our experiments reveal three principal findings. First, demographic prompting is highly model- and task-dependent. The baseline is best in 12 of 25 model–task pairs. Among the remaining 13, 11 peak with 1–3 attributes, while two require higher-order combinations. The full-attribute prompt is never the best configuration. Second, the overall magnitude of demographic influence on human annotations does not predict which attributes improve LLM alignment; instead, both the learnability and the directional coherence of each attribute’s annotation signal need to be considered jointly. Third, neuron probing reveals that specialized activation correlates with alignment gains only under coherent annotation signals, and that activation volume alone does not imply steerability. Together, these results demonstrate that demographic prompting is not a monolithic intervention: its utility is highly context-dependent, shaped by attribute signal quality, task characteristics, and model architecture.</abstract>
<identifier type="citekey">kamruzzaman-etal-2026-demographic</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.60/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1373</start>
<end>1399</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Demographic Prompting at Scale: When More Attributes Hurt LLM–Human Agreement
%A Kamruzzaman, Mahammed
%A Das, Shrabon Kumar
%A Kim, Gene Louis
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F kamruzzaman-etal-2026-demographic
%X We investigate how annotator demographic attributes, supplied as prompt cues, shape the alignment between large language model (LLM) predictions and human annotations across five tasks. Using five open-source LLMs, we systematically vary the number and composition of demographic components in the prompt, spanning every combination from single-attribute through full-attribute configurations. Our experiments reveal three principal findings. First, demographic prompting is highly model- and task-dependent. The baseline is best in 12 of 25 model–task pairs. Among the remaining 13, 11 peak with 1–3 attributes, while two require higher-order combinations. The full-attribute prompt is never the best configuration. Second, the overall magnitude of demographic influence on human annotations does not predict which attributes improve LLM alignment; instead, both the learnability and the directional coherence of each attribute’s annotation signal need to be considered jointly. Third, neuron probing reveals that specialized activation correlates with alignment gains only under coherent annotation signals, and that activation volume alone does not imply steerability. Together, these results demonstrate that demographic prompting is not a monolithic intervention: its utility is highly context-dependent, shaped by attribute signal quality, task characteristics, and model architecture.
%U https://aclanthology.org/2026.emnlp-main.60/
%P 1373-1399
Markdown (Informal)
[Demographic Prompting at Scale: When More Attributes Hurt LLM–Human Agreement](https://aclanthology.org/2026.emnlp-main.60/) (Kamruzzaman et al., EMNLP 2026)
ACL