@inproceedings{cao-etal-2026-procedural,
title = "Procedural Blind Spots in Agent Evaluation: A Survey, Coverage Analysis, and New Missions for {NLP}",
author = "Cao, Hongliu and
Driouich, Ilias and
Thomas, Eoin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1162/",
pages = "24601--24633",
ISBN = "979-8-89176-462-0",
abstract = "Large Language Model (LLM) agents increasingly operate in multi-step, multi-turn settings that combine user interaction, tool use, state changes, and communication. Most agent evaluation still emphasizes task completion, while procedure-aware checks remain fragmented. We conduct a systematic survey and introduce $\mathcal{F}^*$, a systematic organizing map that maps scattered procedural checks into a shared audit space spanning integrity relations and quality dimensions. Applying this framework to recent agent-evaluation work reveals three procedural blind spots: most work remains outcome-focused or procedurally under-specified, procedure-aware checks mostly remain local spot-checks, and communication grounding lags action compliance. To close these gaps, we propose four new Natural Language Processing (NLP) missions around communication grounding, constraint semantics, reasoning faithfulness, and procedural equity. This survey provides a comprehensive overview of procedure-aware agent evaluation, offering a common vocabulary and coverage map that clarify what current benchmarks can certify and provide clear guidelines for future research."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="cao-etal-2026-procedural">
<titleInfo>
<title>Procedural Blind Spots in Agent Evaluation: A Survey, Coverage Analysis, and New Missions for NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hongliu</namePart>
<namePart type="family">Cao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ilias</namePart>
<namePart type="family">Driouich</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Eoin</namePart>
<namePart type="family">Thomas</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large Language Model (LLM) agents increasingly operate in multi-step, multi-turn settings that combine user interaction, tool use, state changes, and communication. Most agent evaluation still emphasizes task completion, while procedure-aware checks remain fragmented. We conduct a systematic survey and introduce \mathcalF^*, a systematic organizing map that maps scattered procedural checks into a shared audit space spanning integrity relations and quality dimensions. Applying this framework to recent agent-evaluation work reveals three procedural blind spots: most work remains outcome-focused or procedurally under-specified, procedure-aware checks mostly remain local spot-checks, and communication grounding lags action compliance. To close these gaps, we propose four new Natural Language Processing (NLP) missions around communication grounding, constraint semantics, reasoning faithfulness, and procedural equity. This survey provides a comprehensive overview of procedure-aware agent evaluation, offering a common vocabulary and coverage map that clarify what current benchmarks can certify and provide clear guidelines for future research.</abstract>
<identifier type="citekey">cao-etal-2026-procedural</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1162/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>24601</start>
<end>24633</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Procedural Blind Spots in Agent Evaluation: A Survey, Coverage Analysis, and New Missions for NLP
%A Cao, Hongliu
%A Driouich, Ilias
%A Thomas, Eoin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F cao-etal-2026-procedural
%X Large Language Model (LLM) agents increasingly operate in multi-step, multi-turn settings that combine user interaction, tool use, state changes, and communication. Most agent evaluation still emphasizes task completion, while procedure-aware checks remain fragmented. We conduct a systematic survey and introduce \mathcalF^*, a systematic organizing map that maps scattered procedural checks into a shared audit space spanning integrity relations and quality dimensions. Applying this framework to recent agent-evaluation work reveals three procedural blind spots: most work remains outcome-focused or procedurally under-specified, procedure-aware checks mostly remain local spot-checks, and communication grounding lags action compliance. To close these gaps, we propose four new Natural Language Processing (NLP) missions around communication grounding, constraint semantics, reasoning faithfulness, and procedural equity. This survey provides a comprehensive overview of procedure-aware agent evaluation, offering a common vocabulary and coverage map that clarify what current benchmarks can certify and provide clear guidelines for future research.
%U https://aclanthology.org/2026.findings-emnlp.1162/
%P 24601-24633
Markdown (Informal)
[Procedural Blind Spots in Agent Evaluation: A Survey, Coverage Analysis, and New Missions for NLP](https://aclanthology.org/2026.findings-emnlp.1162/) (Cao et al., Findings 2026)
ACL