@inproceedings{ryu-etal-2026-evaluating,
title = "Evaluating {LLM} Agents Beyond Local Edit Validity",
author = "Ryu, Kunhee and
Lee, Chi-Guhn and
Lee, Keeheon",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.893/",
pages = "18910--18944",
ISBN = "979-8-89176-462-0",
abstract = "Evaluations of sequential large language model (LLM) agents often report local action validity or final success. Delayed constructive tasks additionally require recoverability, meaning each accepted edit must leave the partial artifact completable within the remaining budget. We introduce a measurement protocol across Word Ladder, an alloy composition proxy, and the fitness landscape of the B1 domain of protein G (GB1). We call an episode surface clean when every action is well formed and locally admissible. Across six models, Word Ladder exposes interface failure with 36.1{\%} local validity. Alloy and GB1 episodes are often surface clean, yet only 16.0{\%} and 6.8{\%} of them succeed. Exact enumeration of 625 GB1 variants over budgets from zero to six decomposes failure. Among surface clean episodes, 51.4{\%} cross a recoverability boundary, while 14.1{\%} of the remaining episodes succeed. Bounded composition audits and intervention checks connect bottlenecks to repair classes. These results establish recoverability as a decomposable and auditable evaluation axis that informs interventions for sequential agents."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ryu-etal-2026-evaluating">
<titleInfo>
<title>Evaluating LLM Agents Beyond Local Edit Validity</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kunhee</namePart>
<namePart type="family">Ryu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chi-Guhn</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keeheon</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Evaluations of sequential large language model (LLM) agents often report local action validity or final success. Delayed constructive tasks additionally require recoverability, meaning each accepted edit must leave the partial artifact completable within the remaining budget. We introduce a measurement protocol across Word Ladder, an alloy composition proxy, and the fitness landscape of the B1 domain of protein G (GB1). We call an episode surface clean when every action is well formed and locally admissible. Across six models, Word Ladder exposes interface failure with 36.1% local validity. Alloy and GB1 episodes are often surface clean, yet only 16.0% and 6.8% of them succeed. Exact enumeration of 625 GB1 variants over budgets from zero to six decomposes failure. Among surface clean episodes, 51.4% cross a recoverability boundary, while 14.1% of the remaining episodes succeed. Bounded composition audits and intervention checks connect bottlenecks to repair classes. These results establish recoverability as a decomposable and auditable evaluation axis that informs interventions for sequential agents.</abstract>
<identifier type="citekey">ryu-etal-2026-evaluating</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.893/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>18910</start>
<end>18944</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating LLM Agents Beyond Local Edit Validity
%A Ryu, Kunhee
%A Lee, Chi-Guhn
%A Lee, Keeheon
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F ryu-etal-2026-evaluating
%X Evaluations of sequential large language model (LLM) agents often report local action validity or final success. Delayed constructive tasks additionally require recoverability, meaning each accepted edit must leave the partial artifact completable within the remaining budget. We introduce a measurement protocol across Word Ladder, an alloy composition proxy, and the fitness landscape of the B1 domain of protein G (GB1). We call an episode surface clean when every action is well formed and locally admissible. Across six models, Word Ladder exposes interface failure with 36.1% local validity. Alloy and GB1 episodes are often surface clean, yet only 16.0% and 6.8% of them succeed. Exact enumeration of 625 GB1 variants over budgets from zero to six decomposes failure. Among surface clean episodes, 51.4% cross a recoverability boundary, while 14.1% of the remaining episodes succeed. Bounded composition audits and intervention checks connect bottlenecks to repair classes. These results establish recoverability as a decomposable and auditable evaluation axis that informs interventions for sequential agents.
%U https://aclanthology.org/2026.findings-emnlp.893/
%P 18910-18944
Markdown (Informal)
[Evaluating LLM Agents Beyond Local Edit Validity](https://aclanthology.org/2026.findings-emnlp.893/) (Ryu et al., Findings 2026)
ACL
- Kunhee Ryu, Chi-Guhn Lee, and Keeheon Lee. 2026. Evaluating LLM Agents Beyond Local Edit Validity. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 18910–18944, Budapest, Hungary. Association for Computational Linguistics.