@inproceedings{lee-etal-2026-interventionbench,
title = "{I}ntervention{B}ench: Can {LLM} Agents Adapt to Mid-task User Interventions?",
author = "Lee, Jaehoon and
Bae, Suyoung and
Lee, Jin-Seop and
Choi, YunSeok and
Lee, Jee-Hyong",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.698/",
doi = "10.18653/2026.findings-emnlp.698",
pages = "14646--14666",
ISBN = "979-8-89176-462-0",
abstract = "LLM agents are increasingly deployed on long-horizon tasks, where a user often revises the goal partway through rather than stating it once at the start. Adapting an ongoing trajectory to such a mid-task intervention is a basic requirement for agents that work with a user, yet existing agentic benchmarks evaluate agents under a goal that stays fixed for the whole task. To address this gap, we introduce \textbf{InterventionBench}, a benchmark that measures how well an agent adapts to a mid-task intervention. Built on AppWorld and WebArena, InterventionBench inserts a user message that revises the goal partway through each task, and defines the revision along three operations and three components of the goal, forming nine intervention types that locate which kind of revision an agent fails on. The released benchmark contains 2,250 intervention instances. We evaluate twelve LLM agents and find that every intervention type degrades performance. Add interventions, which extend the goal rather than replace it, are consistently the hardest. These results establish adapting to a mid-task intervention as a substantial and open challenge for current LLM agents. Our code and data are available at https://github.com/jhn25/InterventionBench."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lee-etal-2026-interventionbench">
<titleInfo>
<title>InterventionBench: Can LLM Agents Adapt to Mid-task User Interventions?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jaehoon</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suyoung</namePart>
<namePart type="family">Bae</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jin-Seop</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">YunSeok</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jee-Hyong</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>LLM agents are increasingly deployed on long-horizon tasks, where a user often revises the goal partway through rather than stating it once at the start. Adapting an ongoing trajectory to such a mid-task intervention is a basic requirement for agents that work with a user, yet existing agentic benchmarks evaluate agents under a goal that stays fixed for the whole task. To address this gap, we introduce InterventionBench, a benchmark that measures how well an agent adapts to a mid-task intervention. Built on AppWorld and WebArena, InterventionBench inserts a user message that revises the goal partway through each task, and defines the revision along three operations and three components of the goal, forming nine intervention types that locate which kind of revision an agent fails on. The released benchmark contains 2,250 intervention instances. We evaluate twelve LLM agents and find that every intervention type degrades performance. Add interventions, which extend the goal rather than replace it, are consistently the hardest. These results establish adapting to a mid-task intervention as a substantial and open challenge for current LLM agents. Our code and data are available at https://github.com/jhn25/InterventionBench.</abstract>
<identifier type="citekey">lee-etal-2026-interventionbench</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.698</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.698/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>14646</start>
<end>14666</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T InterventionBench: Can LLM Agents Adapt to Mid-task User Interventions?
%A Lee, Jaehoon
%A Bae, Suyoung
%A Lee, Jin-Seop
%A Choi, YunSeok
%A Lee, Jee-Hyong
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F lee-etal-2026-interventionbench
%X LLM agents are increasingly deployed on long-horizon tasks, where a user often revises the goal partway through rather than stating it once at the start. Adapting an ongoing trajectory to such a mid-task intervention is a basic requirement for agents that work with a user, yet existing agentic benchmarks evaluate agents under a goal that stays fixed for the whole task. To address this gap, we introduce InterventionBench, a benchmark that measures how well an agent adapts to a mid-task intervention. Built on AppWorld and WebArena, InterventionBench inserts a user message that revises the goal partway through each task, and defines the revision along three operations and three components of the goal, forming nine intervention types that locate which kind of revision an agent fails on. The released benchmark contains 2,250 intervention instances. We evaluate twelve LLM agents and find that every intervention type degrades performance. Add interventions, which extend the goal rather than replace it, are consistently the hardest. These results establish adapting to a mid-task intervention as a substantial and open challenge for current LLM agents. Our code and data are available at https://github.com/jhn25/InterventionBench.
%R 10.18653/2026.findings-emnlp.698
%U https://aclanthology.org/2026.findings-emnlp.698/
%U https://doi.org/10.18653/2026.findings-emnlp.698
%P 14646-14666
Markdown (Informal)
[InterventionBench: Can LLM Agents Adapt to Mid-task User Interventions?](https://aclanthology.org/2026.findings-emnlp.698/) (Lee et al., Findings 2026)
ACL