@inproceedings{jones-kasserra-2026-conjoint,
title = "Conjoint Auditing: A Framework for Measuring the Social Impact of {LLM} Deployment",
author = "Jones, Katrina M. and
Kasserra, Lennart",
editor = "Atwell, Katherine and
Borah, Angana and
Dementieva, Daryna and
Kreiss, Elisa and
Kotonya, Neema and
Liu, Jiarui and
Olson, Liz and
Wan, Ruyuan and
Zhao, Jieyu",
booktitle = "Proceedings of the Fifth Workshop on {NLP} for Positive Impact ({NLP}4{PI})",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.nlp4pi-1.12/",
pages = "125--139",
ISBN = "979-8-89176-495-8",
abstract = "How can we rigorously measure the social impact of LLMs in high-stakes deployments? Existing bias benchmarks flag problems at the model level but offer limited insight into how bias manifests across contexts, magnitudes, attributes, and their interactions. We propose conjoint auditing, a framework that adapts conjoint experimental design from market research to LLM bias evaluation. By presenting randomly generated multi-attribute profiles in a simulated deployment scenario, it delivers deployment-specific estimates of attribute effects and their interactions. Applying the framework for a financial analyst hiring scenario across two commercial LLMs (GPT-5.4 mini and Claude Haiku 4.5; roughly 40,000 observations each), we find that protected attributes exert small but statistically significant and substantively meaningful effects (1-2{\%} of explained variance), with directions and magnitudes varying sharply by model. We also detect clear intersectional patterns, such as amplified disability penalties for women and reduced returns to experience for Black candidates."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="jones-kasserra-2026-conjoint">
<titleInfo>
<title>Conjoint Auditing: A Framework for Measuring the Social Impact of LLM Deployment</title>
</titleInfo>
<name type="personal">
<namePart type="given">Katrina</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Jones</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lennart</namePart>
<namePart type="family">Kasserra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Fifth Workshop on NLP for Positive Impact (NLP4PI)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Katherine</namePart>
<namePart type="family">Atwell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Angana</namePart>
<namePart type="family">Borah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daryna</namePart>
<namePart type="family">Dementieva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elisa</namePart>
<namePart type="family">Kreiss</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Neema</namePart>
<namePart type="family">Kotonya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiarui</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Liz</namePart>
<namePart type="family">Olson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruyuan</namePart>
<namePart type="family">Wan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jieyu</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-495-8</identifier>
</relatedItem>
<abstract>How can we rigorously measure the social impact of LLMs in high-stakes deployments? Existing bias benchmarks flag problems at the model level but offer limited insight into how bias manifests across contexts, magnitudes, attributes, and their interactions. We propose conjoint auditing, a framework that adapts conjoint experimental design from market research to LLM bias evaluation. By presenting randomly generated multi-attribute profiles in a simulated deployment scenario, it delivers deployment-specific estimates of attribute effects and their interactions. Applying the framework for a financial analyst hiring scenario across two commercial LLMs (GPT-5.4 mini and Claude Haiku 4.5; roughly 40,000 observations each), we find that protected attributes exert small but statistically significant and substantively meaningful effects (1-2% of explained variance), with directions and magnitudes varying sharply by model. We also detect clear intersectional patterns, such as amplified disability penalties for women and reduced returns to experience for Black candidates.</abstract>
<identifier type="citekey">jones-kasserra-2026-conjoint</identifier>
<location>
<url>https://aclanthology.org/2026.nlp4pi-1.12/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>125</start>
<end>139</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Conjoint Auditing: A Framework for Measuring the Social Impact of LLM Deployment
%A Jones, Katrina M.
%A Kasserra, Lennart
%Y Atwell, Katherine
%Y Borah, Angana
%Y Dementieva, Daryna
%Y Kreiss, Elisa
%Y Kotonya, Neema
%Y Liu, Jiarui
%Y Olson, Liz
%Y Wan, Ruyuan
%Y Zhao, Jieyu
%S Proceedings of the Fifth Workshop on NLP for Positive Impact (NLP4PI)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-495-8
%F jones-kasserra-2026-conjoint
%X How can we rigorously measure the social impact of LLMs in high-stakes deployments? Existing bias benchmarks flag problems at the model level but offer limited insight into how bias manifests across contexts, magnitudes, attributes, and their interactions. We propose conjoint auditing, a framework that adapts conjoint experimental design from market research to LLM bias evaluation. By presenting randomly generated multi-attribute profiles in a simulated deployment scenario, it delivers deployment-specific estimates of attribute effects and their interactions. Applying the framework for a financial analyst hiring scenario across two commercial LLMs (GPT-5.4 mini and Claude Haiku 4.5; roughly 40,000 observations each), we find that protected attributes exert small but statistically significant and substantively meaningful effects (1-2% of explained variance), with directions and magnitudes varying sharply by model. We also detect clear intersectional patterns, such as amplified disability penalties for women and reduced returns to experience for Black candidates.
%U https://aclanthology.org/2026.nlp4pi-1.12/
%P 125-139
Markdown (Informal)
[Conjoint Auditing: A Framework for Measuring the Social Impact of LLM Deployment](https://aclanthology.org/2026.nlp4pi-1.12/) (Jones & Kasserra, NLP4PI 2026)
ACL