@inproceedings{putri-2026-live,
title = "Does Live Browser Context Improve {LLM}-Assisted End-to-End Test Script Repair? An Empirical Study of Playwright {MCP}",
author = "Putri, Divi Galih Prasetyo",
editor = "Zhang, Chen and
Allaway, Emily and
M{'}hamdi, Meryem and
Limkonchotiwat, Peerat and
Bai, Richard He and
T.Y.S.S., Santosh and
Thapa, Surendrabikram and
Li, Yinqiao",
booktitle = "Proceedings of the 10th Widening {NLP} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.winlp-main.9/",
pages = "87--95",
ISBN = "979-8-89176-472-9",
abstract = "Automated End-to-End test scripts are inherently fragile. Modifications to the User Interface (UI) cause automation test script failures, imposing significant maintenance overhead. Automated test script repair using Large Language Model (LLM) has shown promise, but existing approaches operate without real-time access to the application under test. To the best of our knowledge, no prior controlled study has compared LLM-based test-script repair with and without MCP-mediated live browser access while keeping the remaining repair settings consistent. Through a controlled experiment on a web application, we inject 30 failure scenarios across two categories of UI modifications (locator changes and DOM structural changes) and evaluate two LLM models (GPT 5.0 Mini and Claude Haiku 4.5) with two conditions (with and without MCP access). Configurations with MCP achieve a correct patch rate of 65{\%}, compared with 25{\%} without MCP. The plausible patch rate reached 98{\%} (vs. 87{\%}), the failure rate dropped from 13.3{\%} to 1.7{\%}, and MTTR decreased by 31 seconds. These results suggest that providing agents with MCP-mediated live browser access primarily enhances repair correctness rather than merely increasing re-execution success rates."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="putri-2026-live">
<titleInfo>
<title>Does Live Browser Context Improve LLM-Assisted End-to-End Test Script Repair? An Empirical Study of Playwright MCP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Divi</namePart>
<namePart type="given">Galih</namePart>
<namePart type="given">Prasetyo</namePart>
<namePart type="family">Putri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 10th Widening NLP Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Chen</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Emily</namePart>
<namePart type="family">Allaway</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Meryem</namePart>
<namePart type="family">M’hamdi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peerat</namePart>
<namePart type="family">Limkonchotiwat</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Richard</namePart>
<namePart type="given">He</namePart>
<namePart type="family">Bai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Santosh</namePart>
<namePart type="family">T.Y.S.S.</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Surendrabikram</namePart>
<namePart type="family">Thapa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yinqiao</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-472-9</identifier>
</relatedItem>
<abstract>Automated End-to-End test scripts are inherently fragile. Modifications to the User Interface (UI) cause automation test script failures, imposing significant maintenance overhead. Automated test script repair using Large Language Model (LLM) has shown promise, but existing approaches operate without real-time access to the application under test. To the best of our knowledge, no prior controlled study has compared LLM-based test-script repair with and without MCP-mediated live browser access while keeping the remaining repair settings consistent. Through a controlled experiment on a web application, we inject 30 failure scenarios across two categories of UI modifications (locator changes and DOM structural changes) and evaluate two LLM models (GPT 5.0 Mini and Claude Haiku 4.5) with two conditions (with and without MCP access). Configurations with MCP achieve a correct patch rate of 65%, compared with 25% without MCP. The plausible patch rate reached 98% (vs. 87%), the failure rate dropped from 13.3% to 1.7%, and MTTR decreased by 31 seconds. These results suggest that providing agents with MCP-mediated live browser access primarily enhances repair correctness rather than merely increasing re-execution success rates.</abstract>
<identifier type="citekey">putri-2026-live</identifier>
<location>
<url>https://aclanthology.org/2026.winlp-main.9/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>87</start>
<end>95</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Does Live Browser Context Improve LLM-Assisted End-to-End Test Script Repair? An Empirical Study of Playwright MCP
%A Putri, Divi Galih Prasetyo
%Y Zhang, Chen
%Y Allaway, Emily
%Y M’hamdi, Meryem
%Y Limkonchotiwat, Peerat
%Y Bai, Richard He
%Y T.Y.S.S., Santosh
%Y Thapa, Surendrabikram
%Y Li, Yinqiao
%S Proceedings of the 10th Widening NLP Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-472-9
%F putri-2026-live
%X Automated End-to-End test scripts are inherently fragile. Modifications to the User Interface (UI) cause automation test script failures, imposing significant maintenance overhead. Automated test script repair using Large Language Model (LLM) has shown promise, but existing approaches operate without real-time access to the application under test. To the best of our knowledge, no prior controlled study has compared LLM-based test-script repair with and without MCP-mediated live browser access while keeping the remaining repair settings consistent. Through a controlled experiment on a web application, we inject 30 failure scenarios across two categories of UI modifications (locator changes and DOM structural changes) and evaluate two LLM models (GPT 5.0 Mini and Claude Haiku 4.5) with two conditions (with and without MCP access). Configurations with MCP achieve a correct patch rate of 65%, compared with 25% without MCP. The plausible patch rate reached 98% (vs. 87%), the failure rate dropped from 13.3% to 1.7%, and MTTR decreased by 31 seconds. These results suggest that providing agents with MCP-mediated live browser access primarily enhances repair correctness rather than merely increasing re-execution success rates.
%U https://aclanthology.org/2026.winlp-main.9/
%P 87-95
Markdown (Informal)
[Does Live Browser Context Improve LLM-Assisted End-to-End Test Script Repair? An Empirical Study of Playwright MCP](https://aclanthology.org/2026.winlp-main.9/) (Putri, WiNLP 2026)
ACL