@inproceedings{driouich-etal-2026-evaluating,
title = "Evaluating Demographics Effect in Policy-Governed Tool-Using Agents Across Real-World Service Domains",
author = "Driouich, Ilias and
Cao, Hongliu and
Thomas, Eoin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1157/",
pages = "24471--24492",
ISBN = "979-8-89176-462-0",
abstract = "Imagine two users contacting the same AI travel agent on the same day, making the same request under identical policy conditions, differing only in the demographic information they provide. The governing policy is explicit and contains no demographic distinction. Should the agent respond identically? We present an empirical study of how demographic context influences the behaviour of policy-governed LLM agents. Building on an existing agentic evaluation framework, we augment policy-governed tasks with controlled demographic cues while keeping policies and task environments fixed, enabling behavioural differences to be attributed solely to demographic signals. To evaluate procedural fairness, we introduce three procedural dimensions: Policy Application Fairness (PAF), Procedural Effort Equity (PEE), and User Fulfillment Parity (UFP), unified into the Demographic Policy Fairness Index (DPFI). Across more than twenty thousand simulated interactions spanning multiple service domains and frontier models, all evaluated systems exhibit systematic demographic disparities that conventional task-success metrics fail to capture. Our analysis identifies three distinct failure modes with different procedural dynamics, demonstrating that aggregate success metrics can obscure how disparities emerge. Overall, the findings suggest that demographic bias in policy-governed LLM agents manifests through procedural behaviour rather than a single uniform failure pattern. The proposed framework provides a foundation for systematically auditing procedural fairness in policy-driven LLM agents."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="driouich-etal-2026-evaluating">
<titleInfo>
<title>Evaluating Demographics Effect in Policy-Governed Tool-Using Agents Across Real-World Service Domains</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ilias</namePart>
<namePart type="family">Driouich</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hongliu</namePart>
<namePart type="family">Cao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Eoin</namePart>
<namePart type="family">Thomas</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Imagine two users contacting the same AI travel agent on the same day, making the same request under identical policy conditions, differing only in the demographic information they provide. The governing policy is explicit and contains no demographic distinction. Should the agent respond identically? We present an empirical study of how demographic context influences the behaviour of policy-governed LLM agents. Building on an existing agentic evaluation framework, we augment policy-governed tasks with controlled demographic cues while keeping policies and task environments fixed, enabling behavioural differences to be attributed solely to demographic signals. To evaluate procedural fairness, we introduce three procedural dimensions: Policy Application Fairness (PAF), Procedural Effort Equity (PEE), and User Fulfillment Parity (UFP), unified into the Demographic Policy Fairness Index (DPFI). Across more than twenty thousand simulated interactions spanning multiple service domains and frontier models, all evaluated systems exhibit systematic demographic disparities that conventional task-success metrics fail to capture. Our analysis identifies three distinct failure modes with different procedural dynamics, demonstrating that aggregate success metrics can obscure how disparities emerge. Overall, the findings suggest that demographic bias in policy-governed LLM agents manifests through procedural behaviour rather than a single uniform failure pattern. The proposed framework provides a foundation for systematically auditing procedural fairness in policy-driven LLM agents.</abstract>
<identifier type="citekey">driouich-etal-2026-evaluating</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1157/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>24471</start>
<end>24492</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating Demographics Effect in Policy-Governed Tool-Using Agents Across Real-World Service Domains
%A Driouich, Ilias
%A Cao, Hongliu
%A Thomas, Eoin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F driouich-etal-2026-evaluating
%X Imagine two users contacting the same AI travel agent on the same day, making the same request under identical policy conditions, differing only in the demographic information they provide. The governing policy is explicit and contains no demographic distinction. Should the agent respond identically? We present an empirical study of how demographic context influences the behaviour of policy-governed LLM agents. Building on an existing agentic evaluation framework, we augment policy-governed tasks with controlled demographic cues while keeping policies and task environments fixed, enabling behavioural differences to be attributed solely to demographic signals. To evaluate procedural fairness, we introduce three procedural dimensions: Policy Application Fairness (PAF), Procedural Effort Equity (PEE), and User Fulfillment Parity (UFP), unified into the Demographic Policy Fairness Index (DPFI). Across more than twenty thousand simulated interactions spanning multiple service domains and frontier models, all evaluated systems exhibit systematic demographic disparities that conventional task-success metrics fail to capture. Our analysis identifies three distinct failure modes with different procedural dynamics, demonstrating that aggregate success metrics can obscure how disparities emerge. Overall, the findings suggest that demographic bias in policy-governed LLM agents manifests through procedural behaviour rather than a single uniform failure pattern. The proposed framework provides a foundation for systematically auditing procedural fairness in policy-driven LLM agents.
%U https://aclanthology.org/2026.findings-emnlp.1157/
%P 24471-24492
Markdown (Informal)
[Evaluating Demographics Effect in Policy-Governed Tool-Using Agents Across Real-World Service Domains](https://aclanthology.org/2026.findings-emnlp.1157/) (Driouich et al., Findings 2026)
ACL