@inproceedings{tonneau-etal-2026-different,
title = "Different Demographic Cues Yield Inconsistent Conclusions About {LLM} Personalization and Bias",
author = "Tonneau, Manuel and
Sehgal, Neil and
Malhotra, Niyati and
Kazemi, Sharif and
Orozco-Olvera, Victor and
Munoz Boudet, Ana Maria and
Subramanian, Lakshmi and
Fraiberger, Samuel and
Guntuku, Sharath Chandra and
Hofmann, Valentin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2118/",
pages = "48170--48198",
ISBN = "979-8-89176-459-0",
abstract = "Demographic cue-based evaluation is widely used to study how large language models (LLMs) adapt their responses to signaled demographic attributes within and across groups. This approach typically relies on a single cue (e.g., names) as a proxy for group membership, implicitly treating different cues as interchangeable operationalizations of a single underlying identity-conditioned behavior. We test this assumption in realistic advice-seeking interactions spanning 14.8 million prompts, focusing on race and gender in a U.S. context. We find that cues for the same group induce only partially overlapping changes in model responses, yielding inconsistent conclusions about personalization, while bias conclusions are unstable, with both magnitude and direction of group differences varying across cues. We further show that these inconsistencies reflect differences in cue{--}group association strength and linguistic features bundled within cues that shape model responses. Together, our findings suggest that demographic conditioning in LLMs is not a cue-invariant category-level parameter but depends fundamentally on how identity is cued, reflecting responses to linguistic signals rather than stable demographic categories. We therefore call for multi-cue, mechanism-aware evaluations as a foundation for robust and interpretable claims about demographic variation in LLM responses."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="tonneau-etal-2026-different">
<titleInfo>
<title>Different Demographic Cues Yield Inconsistent Conclusions About LLM Personalization and Bias</title>
</titleInfo>
<name type="personal">
<namePart type="given">Manuel</namePart>
<namePart type="family">Tonneau</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Neil</namePart>
<namePart type="family">Sehgal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Niyati</namePart>
<namePart type="family">Malhotra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sharif</namePart>
<namePart type="family">Kazemi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Victor</namePart>
<namePart type="family">Orozco-Olvera</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ana</namePart>
<namePart type="given">Maria</namePart>
<namePart type="family">Munoz Boudet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lakshmi</namePart>
<namePart type="family">Subramanian</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Samuel</namePart>
<namePart type="family">Fraiberger</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sharath</namePart>
<namePart type="given">Chandra</namePart>
<namePart type="family">Guntuku</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Valentin</namePart>
<namePart type="family">Hofmann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Demographic cue-based evaluation is widely used to study how large language models (LLMs) adapt their responses to signaled demographic attributes within and across groups. This approach typically relies on a single cue (e.g., names) as a proxy for group membership, implicitly treating different cues as interchangeable operationalizations of a single underlying identity-conditioned behavior. We test this assumption in realistic advice-seeking interactions spanning 14.8 million prompts, focusing on race and gender in a U.S. context. We find that cues for the same group induce only partially overlapping changes in model responses, yielding inconsistent conclusions about personalization, while bias conclusions are unstable, with both magnitude and direction of group differences varying across cues. We further show that these inconsistencies reflect differences in cue–group association strength and linguistic features bundled within cues that shape model responses. Together, our findings suggest that demographic conditioning in LLMs is not a cue-invariant category-level parameter but depends fundamentally on how identity is cued, reflecting responses to linguistic signals rather than stable demographic categories. We therefore call for multi-cue, mechanism-aware evaluations as a foundation for robust and interpretable claims about demographic variation in LLM responses.</abstract>
<identifier type="citekey">tonneau-etal-2026-different</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2118/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>48170</start>
<end>48198</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Different Demographic Cues Yield Inconsistent Conclusions About LLM Personalization and Bias
%A Tonneau, Manuel
%A Sehgal, Neil
%A Malhotra, Niyati
%A Kazemi, Sharif
%A Orozco-Olvera, Victor
%A Munoz Boudet, Ana Maria
%A Subramanian, Lakshmi
%A Fraiberger, Samuel
%A Guntuku, Sharath Chandra
%A Hofmann, Valentin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F tonneau-etal-2026-different
%X Demographic cue-based evaluation is widely used to study how large language models (LLMs) adapt their responses to signaled demographic attributes within and across groups. This approach typically relies on a single cue (e.g., names) as a proxy for group membership, implicitly treating different cues as interchangeable operationalizations of a single underlying identity-conditioned behavior. We test this assumption in realistic advice-seeking interactions spanning 14.8 million prompts, focusing on race and gender in a U.S. context. We find that cues for the same group induce only partially overlapping changes in model responses, yielding inconsistent conclusions about personalization, while bias conclusions are unstable, with both magnitude and direction of group differences varying across cues. We further show that these inconsistencies reflect differences in cue–group association strength and linguistic features bundled within cues that shape model responses. Together, our findings suggest that demographic conditioning in LLMs is not a cue-invariant category-level parameter but depends fundamentally on how identity is cued, reflecting responses to linguistic signals rather than stable demographic categories. We therefore call for multi-cue, mechanism-aware evaluations as a foundation for robust and interpretable claims about demographic variation in LLM responses.
%U https://aclanthology.org/2026.emnlp-main.2118/
%P 48170-48198
Markdown (Informal)
[Different Demographic Cues Yield Inconsistent Conclusions About LLM Personalization and Bias](https://aclanthology.org/2026.emnlp-main.2118/) (Tonneau et al., EMNLP 2026)
ACL
- Manuel Tonneau, Neil Sehgal, Niyati Malhotra, Sharif Kazemi, Victor Orozco-Olvera, Ana Maria Munoz Boudet, Lakshmi Subramanian, Samuel Fraiberger, Sharath Chandra Guntuku, and Valentin Hofmann. 2026. Different Demographic Cues Yield Inconsistent Conclusions About LLM Personalization and Bias. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 48170–48198, Budapest, Hungary. Association for Computational Linguistics.