@inproceedings{li-etal-2026-sopbench,
title = "{SOPB}ench: Evaluating Language Agents at Following Standard Operating Procedures and Constraints",
author = "Li, Zekun and
Huang, Shinda and
Wang, Jiangtian and
Zhang, Nathan and
Antoniades, Antonis and
Hua, Wenyue and
Zhu, Kaijie and
Zeng, Sirui and
Wang, Chi and
Wang, William Yang and
Yan, Xifeng",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.936/",
pages = "21561--21616",
ISBN = "979-8-89176-459-0",
abstract = "As language agents are increasingly entrusted with tool-mediated actions, reaching the correct outcome is not sufficient: agents must also comply with domain-specific standard operating procedures (SOPs), policies, and constraints throughout execution. We introduce SOPBench, an executable benchmark for evaluating procedural compliance in language agents. SOPBench provides sandbox environments spanning seven customer-service domains, with 167 executable tools, 70 service-specific SOPs, and 830 validated test cases. Its code-based oracle verifiers assess compliance at the outcome, step, and trajectory levels without relying on human or LLM judges. Evaluating 23 leading models, we find that reasoning models perform best but remain far from reliable, particularly in difficult domains and under simple adversarial instructions. Error analysis further reveals distinct procedural failure modes across model capability levels. Finally, a layered harness ablation shows that even when agents are given ground-truth constraint verdicts, 12-38{\%} of cases still fail. This residual gap suggests that a key challenge lies not only in retrieving or verifying constraints, but also in consistently translating verified constraints into compliant actions. We release the code, data, and more than 40k agent trajectories."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-etal-2026-sopbench">
<titleInfo>
<title>SOPBench: Evaluating Language Agents at Following Standard Operating Procedures and Constraints</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zekun</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shinda</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiangtian</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nathan</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Antonis</namePart>
<namePart type="family">Antoniades</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenyue</namePart>
<namePart type="family">Hua</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kaijie</namePart>
<namePart type="family">Zhu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sirui</namePart>
<namePart type="family">Zeng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chi</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">William</namePart>
<namePart type="given">Yang</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xifeng</namePart>
<namePart type="family">Yan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>As language agents are increasingly entrusted with tool-mediated actions, reaching the correct outcome is not sufficient: agents must also comply with domain-specific standard operating procedures (SOPs), policies, and constraints throughout execution. We introduce SOPBench, an executable benchmark for evaluating procedural compliance in language agents. SOPBench provides sandbox environments spanning seven customer-service domains, with 167 executable tools, 70 service-specific SOPs, and 830 validated test cases. Its code-based oracle verifiers assess compliance at the outcome, step, and trajectory levels without relying on human or LLM judges. Evaluating 23 leading models, we find that reasoning models perform best but remain far from reliable, particularly in difficult domains and under simple adversarial instructions. Error analysis further reveals distinct procedural failure modes across model capability levels. Finally, a layered harness ablation shows that even when agents are given ground-truth constraint verdicts, 12-38% of cases still fail. This residual gap suggests that a key challenge lies not only in retrieving or verifying constraints, but also in consistently translating verified constraints into compliant actions. We release the code, data, and more than 40k agent trajectories.</abstract>
<identifier type="citekey">li-etal-2026-sopbench</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.936/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>21561</start>
<end>21616</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SOPBench: Evaluating Language Agents at Following Standard Operating Procedures and Constraints
%A Li, Zekun
%A Huang, Shinda
%A Wang, Jiangtian
%A Zhang, Nathan
%A Antoniades, Antonis
%A Hua, Wenyue
%A Zhu, Kaijie
%A Zeng, Sirui
%A Wang, Chi
%A Wang, William Yang
%A Yan, Xifeng
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F li-etal-2026-sopbench
%X As language agents are increasingly entrusted with tool-mediated actions, reaching the correct outcome is not sufficient: agents must also comply with domain-specific standard operating procedures (SOPs), policies, and constraints throughout execution. We introduce SOPBench, an executable benchmark for evaluating procedural compliance in language agents. SOPBench provides sandbox environments spanning seven customer-service domains, with 167 executable tools, 70 service-specific SOPs, and 830 validated test cases. Its code-based oracle verifiers assess compliance at the outcome, step, and trajectory levels without relying on human or LLM judges. Evaluating 23 leading models, we find that reasoning models perform best but remain far from reliable, particularly in difficult domains and under simple adversarial instructions. Error analysis further reveals distinct procedural failure modes across model capability levels. Finally, a layered harness ablation shows that even when agents are given ground-truth constraint verdicts, 12-38% of cases still fail. This residual gap suggests that a key challenge lies not only in retrieving or verifying constraints, but also in consistently translating verified constraints into compliant actions. We release the code, data, and more than 40k agent trajectories.
%U https://aclanthology.org/2026.emnlp-main.936/
%P 21561-21616
Markdown (Informal)
[SOPBench: Evaluating Language Agents at Following Standard Operating Procedures and Constraints](https://aclanthology.org/2026.emnlp-main.936/) (Li et al., EMNLP 2026)
ACL
- Zekun Li, Shinda Huang, Jiangtian Wang, Nathan Zhang, Antonis Antoniades, Wenyue Hua, Kaijie Zhu, Sirui Zeng, Chi Wang, William Yang Wang, and Xifeng Yan. 2026. SOPBench: Evaluating Language Agents at Following Standard Operating Procedures and Constraints. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 21561–21616, Budapest, Hungary. Association for Computational Linguistics.