@inproceedings{liu-etal-2026-beyond-single,
title = "Beyond Single-Policy: Evaluating Composed Organization-Specific Policy Alignment in {LLM} Chatbots",
author = "Liu, Yingjie and
Hu, Yongxiang and
Wang, Xuan and
Li, Yilun and
Wei, Yunlei and
Wang, Xiaoyu and
Zhou, Yangfan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.254/",
pages = "5767--5786",
ISBN = "979-8-89176-459-0",
abstract = "Large language model chatbots are increasingly deployed in organizational settings such as healthcare, finance, and public services, where organization-specific policies specify allowed and prohibited content. Existing policy-alignment benchmarks construct tests from one policy at a time, leaving failures caused by policy composition under-tested. We present COPAL, an automated framework for evaluating composed-policy alignment in organizational chatbots. COPAL uses empirically derived interaction patterns to generate queries that require multiple policies to be handled in one response, and pairs each query with an explicit handling contract specifying what to provide and avoid. Applied to 30 organization-like company worlds, COPAL exposes substantial composed-policy handling failures. Across 9 served models in our automated evaluation setup, composed-policy requests yield a 33.1{\%} error rate, indicating that composed-policy alignment remains a challenging evaluation target. Our code is available at https://github.com/BruceAllenCS/COPAL-code."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-beyond-single">
<titleInfo>
<title>Beyond Single-Policy: Evaluating Composed Organization-Specific Policy Alignment in LLM Chatbots</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yingjie</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yongxiang</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xuan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yilun</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yunlei</namePart>
<namePart type="family">Wei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaoyu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yangfan</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large language model chatbots are increasingly deployed in organizational settings such as healthcare, finance, and public services, where organization-specific policies specify allowed and prohibited content. Existing policy-alignment benchmarks construct tests from one policy at a time, leaving failures caused by policy composition under-tested. We present COPAL, an automated framework for evaluating composed-policy alignment in organizational chatbots. COPAL uses empirically derived interaction patterns to generate queries that require multiple policies to be handled in one response, and pairs each query with an explicit handling contract specifying what to provide and avoid. Applied to 30 organization-like company worlds, COPAL exposes substantial composed-policy handling failures. Across 9 served models in our automated evaluation setup, composed-policy requests yield a 33.1% error rate, indicating that composed-policy alignment remains a challenging evaluation target. Our code is available at https://github.com/BruceAllenCS/COPAL-code.</abstract>
<identifier type="citekey">liu-etal-2026-beyond-single</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.254/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>5767</start>
<end>5786</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Single-Policy: Evaluating Composed Organization-Specific Policy Alignment in LLM Chatbots
%A Liu, Yingjie
%A Hu, Yongxiang
%A Wang, Xuan
%A Li, Yilun
%A Wei, Yunlei
%A Wang, Xiaoyu
%A Zhou, Yangfan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F liu-etal-2026-beyond-single
%X Large language model chatbots are increasingly deployed in organizational settings such as healthcare, finance, and public services, where organization-specific policies specify allowed and prohibited content. Existing policy-alignment benchmarks construct tests from one policy at a time, leaving failures caused by policy composition under-tested. We present COPAL, an automated framework for evaluating composed-policy alignment in organizational chatbots. COPAL uses empirically derived interaction patterns to generate queries that require multiple policies to be handled in one response, and pairs each query with an explicit handling contract specifying what to provide and avoid. Applied to 30 organization-like company worlds, COPAL exposes substantial composed-policy handling failures. Across 9 served models in our automated evaluation setup, composed-policy requests yield a 33.1% error rate, indicating that composed-policy alignment remains a challenging evaluation target. Our code is available at https://github.com/BruceAllenCS/COPAL-code.
%U https://aclanthology.org/2026.emnlp-main.254/
%P 5767-5786
Markdown (Informal)
[Beyond Single-Policy: Evaluating Composed Organization-Specific Policy Alignment in LLM Chatbots](https://aclanthology.org/2026.emnlp-main.254/) (Liu et al., EMNLP 2026)
ACL