@inproceedings{xue-etal-2026-beyond-final,
title = "Beyond Final Success: Semantic Progress Diagnostics for Agent Trajectory Evaluation",
author = "Xue, Nan and
Wang, Tinghao and
Tang, Li",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2140/",
pages = "48652--48672",
ISBN = "979-8-89176-459-0",
abstract = "Large language model agents are increasingly expected to handle complex, long-horizon tasks that require sustained tool use and stateful interaction. Final outcome labels in existing agent benchmarks provide only a coarse view of long-horizon agent execution. They obscure differences in the execution process among successful trajectories and collapse failed trajectories that may have achieved different amounts of task-relevant progress. We introduce the Auditable Semantic Reference Graph (ASRG), a post-hoc framework that represents task-relevant progress with task-specific Semantic Reference Units (SRUs) and matches trajectories against these references. Each SRU is a checkable task milestone with an explicit verifier. Deterministic matching quantifies process coverage and coverage-weighted first-hit timing, while an action-level diagnostic records actions unexplained by the labeling rules. We apply ASRG to 4,740 Toolathlon evaluation trajectories from 22 models across 108 tasks. ASRG reveals process-coverage differences between models with identical final-success rates and first-support timing differences between same-task trajectories with matched coverage. By relating measured progress to resource use, ASRG also supports process-efficiency analysis. ASRG complements final outcome evaluation with auditable, reference-conditioned process diagnostics."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="xue-etal-2026-beyond-final">
<titleInfo>
<title>Beyond Final Success: Semantic Progress Diagnostics for Agent Trajectory Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nan</namePart>
<namePart type="family">Xue</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tinghao</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Li</namePart>
<namePart type="family">Tang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large language model agents are increasingly expected to handle complex, long-horizon tasks that require sustained tool use and stateful interaction. Final outcome labels in existing agent benchmarks provide only a coarse view of long-horizon agent execution. They obscure differences in the execution process among successful trajectories and collapse failed trajectories that may have achieved different amounts of task-relevant progress. We introduce the Auditable Semantic Reference Graph (ASRG), a post-hoc framework that represents task-relevant progress with task-specific Semantic Reference Units (SRUs) and matches trajectories against these references. Each SRU is a checkable task milestone with an explicit verifier. Deterministic matching quantifies process coverage and coverage-weighted first-hit timing, while an action-level diagnostic records actions unexplained by the labeling rules. We apply ASRG to 4,740 Toolathlon evaluation trajectories from 22 models across 108 tasks. ASRG reveals process-coverage differences between models with identical final-success rates and first-support timing differences between same-task trajectories with matched coverage. By relating measured progress to resource use, ASRG also supports process-efficiency analysis. ASRG complements final outcome evaluation with auditable, reference-conditioned process diagnostics.</abstract>
<identifier type="citekey">xue-etal-2026-beyond-final</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2140/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>48652</start>
<end>48672</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Final Success: Semantic Progress Diagnostics for Agent Trajectory Evaluation
%A Xue, Nan
%A Wang, Tinghao
%A Tang, Li
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F xue-etal-2026-beyond-final
%X Large language model agents are increasingly expected to handle complex, long-horizon tasks that require sustained tool use and stateful interaction. Final outcome labels in existing agent benchmarks provide only a coarse view of long-horizon agent execution. They obscure differences in the execution process among successful trajectories and collapse failed trajectories that may have achieved different amounts of task-relevant progress. We introduce the Auditable Semantic Reference Graph (ASRG), a post-hoc framework that represents task-relevant progress with task-specific Semantic Reference Units (SRUs) and matches trajectories against these references. Each SRU is a checkable task milestone with an explicit verifier. Deterministic matching quantifies process coverage and coverage-weighted first-hit timing, while an action-level diagnostic records actions unexplained by the labeling rules. We apply ASRG to 4,740 Toolathlon evaluation trajectories from 22 models across 108 tasks. ASRG reveals process-coverage differences between models with identical final-success rates and first-support timing differences between same-task trajectories with matched coverage. By relating measured progress to resource use, ASRG also supports process-efficiency analysis. ASRG complements final outcome evaluation with auditable, reference-conditioned process diagnostics.
%U https://aclanthology.org/2026.emnlp-main.2140/
%P 48652-48672
Markdown (Informal)
[Beyond Final Success: Semantic Progress Diagnostics for Agent Trajectory Evaluation](https://aclanthology.org/2026.emnlp-main.2140/) (Xue et al., EMNLP 2026)
ACL