@inproceedings{podivilov-etal-2026-agentlens,
title = "{A}gent{L}ens: Trajectory Reviews for Coding Agent Evaluation",
author = "Podivilov, Andrey and
Lomshakov, Vadim and
Savin, Sergey and
Startsev, Matvey and
Pozharskii, Roman and
Parshin, Maksim and
Nikolenko, Sergey",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.94/",
pages = "1588--1612",
ISBN = "979-8-89176-460-6",
abstract = "We present AgentLens, a trajectory-level evaluation framework for interactive code agents, built from production-informed tasks and integrated into continuous regression testing. Most code-agent benchmarks reduce a run to a single bit{---}did the task pass?{---}but the people who actually use these agents experience the entire trajectory: how the agent follows instructions, uses its tools, verifies its own work, recovers from mistakes, and talks to them along the way. AgentLens evaluates that whole trajectory, pairing formal verification with LLM-written trajectory reviews and side-by-side comparisons, so that each run yields a readable explanation of the final score. Apart from ranking models, AgentLens can be used to diagnose model behavior, compare successive versions of agents, and catch product regressions in an evaluation pipeline. We release AgentLens at https://github.com/agent-lens/agent-lens-bench."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="podivilov-etal-2026-agentlens">
<titleInfo>
<title>AgentLens: Trajectory Reviews for Coding Agent Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Andrey</namePart>
<namePart type="family">Podivilov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vadim</namePart>
<namePart type="family">Lomshakov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sergey</namePart>
<namePart type="family">Savin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Matvey</namePart>
<namePart type="family">Startsev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Roman</namePart>
<namePart type="family">Pozharskii</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maksim</namePart>
<namePart type="family">Parshin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sergey</namePart>
<namePart type="family">Nikolenko</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>We present AgentLens, a trajectory-level evaluation framework for interactive code agents, built from production-informed tasks and integrated into continuous regression testing. Most code-agent benchmarks reduce a run to a single bit—did the task pass?—but the people who actually use these agents experience the entire trajectory: how the agent follows instructions, uses its tools, verifies its own work, recovers from mistakes, and talks to them along the way. AgentLens evaluates that whole trajectory, pairing formal verification with LLM-written trajectory reviews and side-by-side comparisons, so that each run yields a readable explanation of the final score. Apart from ranking models, AgentLens can be used to diagnose model behavior, compare successive versions of agents, and catch product regressions in an evaluation pipeline. We release AgentLens at https://github.com/agent-lens/agent-lens-bench.</abstract>
<identifier type="citekey">podivilov-etal-2026-agentlens</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.94/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1588</start>
<end>1612</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T AgentLens: Trajectory Reviews for Coding Agent Evaluation
%A Podivilov, Andrey
%A Lomshakov, Vadim
%A Savin, Sergey
%A Startsev, Matvey
%A Pozharskii, Roman
%A Parshin, Maksim
%A Nikolenko, Sergey
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F podivilov-etal-2026-agentlens
%X We present AgentLens, a trajectory-level evaluation framework for interactive code agents, built from production-informed tasks and integrated into continuous regression testing. Most code-agent benchmarks reduce a run to a single bit—did the task pass?—but the people who actually use these agents experience the entire trajectory: how the agent follows instructions, uses its tools, verifies its own work, recovers from mistakes, and talks to them along the way. AgentLens evaluates that whole trajectory, pairing formal verification with LLM-written trajectory reviews and side-by-side comparisons, so that each run yields a readable explanation of the final score. Apart from ranking models, AgentLens can be used to diagnose model behavior, compare successive versions of agents, and catch product regressions in an evaluation pipeline. We release AgentLens at https://github.com/agent-lens/agent-lens-bench.
%U https://aclanthology.org/2026.emnlp-industry.94/
%P 1588-1612
Markdown (Informal)
[AgentLens: Trajectory Reviews for Coding Agent Evaluation](https://aclanthology.org/2026.emnlp-industry.94/) (Podivilov et al., EMNLP 2026)
ACL
- Andrey Podivilov, Vadim Lomshakov, Sergey Savin, Matvey Startsev, Roman Pozharskii, Maksim Parshin, and Sergey Nikolenko. 2026. AgentLens: Trajectory Reviews for Coding Agent Evaluation. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track, pages 1588–1612, Budapest, Hungary. Association for Computational Linguistics.