@inproceedings{lei-etal-2026-webcompass,
title = "{W}eb{C}ompass: Evaluating Multimodal Code Agents in Web Development from Vision, Instruction, and Interaction to Implementation",
author = "Lei, Xinping and
Che, Xinyu and
Xiong, Junqi and
Zhang, Chenchen and
Huang, Yukai and
Zhou, Chenyu and
Huang, Haoyang and
Liu, Minghao and
Zhu, Letian and
Ye, Hongyi and
Deng, Ken and
Zhan, Zizheng and
Li, Han and
Yao, Yifan and
Sun, Ming and
Zhang, Zhaoxiang and
Liu, Jiaheng",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1669/",
pages = "37948--37984",
ISBN = "979-8-89176-459-0",
abstract = "Large language models are increasingly deployed as interactive coding agents for end-to-end web development, yet existing benchmarks evaluate narrow slices of this workflow{---}typically text-conditioned generation with static correctness metrics{---}overlooking visual fidelity, interactivity, and codebase-level reasoning. We introduce WebCompass, a benchmark that covers the full web development lifecycle by spanning three input modalities (text, image, video) and three task types (generation, editing, repair). To match the diverse nature of these tasks, we pair checklist-guided LLM-as-a-Judge for patch-based evaluation with a novel Agent-as-a-Judge protocol that executes generated websites in a real browser and synthesizes targeted test cases. Experiments on ten models reveal a 25-point gap between the best closed- and open-source models, persistent weaknesses in visual aesthetics even among frontier systems, and divergent performance profiles across task types that challenge the notion of a single ``best'' web coding agent."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lei-etal-2026-webcompass">
<titleInfo>
<title>WebCompass: Evaluating Multimodal Code Agents in Web Development from Vision, Instruction, and Interaction to Implementation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xinping</namePart>
<namePart type="family">Lei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="family">Che</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junqi</namePart>
<namePart type="family">Xiong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chenchen</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yukai</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chenyu</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haoyang</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghao</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Letian</namePart>
<namePart type="family">Zhu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hongyi</namePart>
<namePart type="family">Ye</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ken</namePart>
<namePart type="family">Deng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zizheng</namePart>
<namePart type="family">Zhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Han</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yifan</namePart>
<namePart type="family">Yao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ming</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhaoxiang</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiaheng</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large language models are increasingly deployed as interactive coding agents for end-to-end web development, yet existing benchmarks evaluate narrow slices of this workflow—typically text-conditioned generation with static correctness metrics—overlooking visual fidelity, interactivity, and codebase-level reasoning. We introduce WebCompass, a benchmark that covers the full web development lifecycle by spanning three input modalities (text, image, video) and three task types (generation, editing, repair). To match the diverse nature of these tasks, we pair checklist-guided LLM-as-a-Judge for patch-based evaluation with a novel Agent-as-a-Judge protocol that executes generated websites in a real browser and synthesizes targeted test cases. Experiments on ten models reveal a 25-point gap between the best closed- and open-source models, persistent weaknesses in visual aesthetics even among frontier systems, and divergent performance profiles across task types that challenge the notion of a single “best” web coding agent.</abstract>
<identifier type="citekey">lei-etal-2026-webcompass</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1669/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>37948</start>
<end>37984</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T WebCompass: Evaluating Multimodal Code Agents in Web Development from Vision, Instruction, and Interaction to Implementation
%A Lei, Xinping
%A Che, Xinyu
%A Xiong, Junqi
%A Zhang, Chenchen
%A Huang, Yukai
%A Zhou, Chenyu
%A Huang, Haoyang
%A Liu, Minghao
%A Zhu, Letian
%A Ye, Hongyi
%A Deng, Ken
%A Zhan, Zizheng
%A Li, Han
%A Yao, Yifan
%A Sun, Ming
%A Zhang, Zhaoxiang
%A Liu, Jiaheng
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F lei-etal-2026-webcompass
%X Large language models are increasingly deployed as interactive coding agents for end-to-end web development, yet existing benchmarks evaluate narrow slices of this workflow—typically text-conditioned generation with static correctness metrics—overlooking visual fidelity, interactivity, and codebase-level reasoning. We introduce WebCompass, a benchmark that covers the full web development lifecycle by spanning three input modalities (text, image, video) and three task types (generation, editing, repair). To match the diverse nature of these tasks, we pair checklist-guided LLM-as-a-Judge for patch-based evaluation with a novel Agent-as-a-Judge protocol that executes generated websites in a real browser and synthesizes targeted test cases. Experiments on ten models reveal a 25-point gap between the best closed- and open-source models, persistent weaknesses in visual aesthetics even among frontier systems, and divergent performance profiles across task types that challenge the notion of a single “best” web coding agent.
%U https://aclanthology.org/2026.emnlp-main.1669/
%P 37948-37984
Markdown (Informal)
[WebCompass: Evaluating Multimodal Code Agents in Web Development from Vision, Instruction, and Interaction to Implementation](https://aclanthology.org/2026.emnlp-main.1669/) (Lei et al., EMNLP 2026)
ACL
- Xinping Lei, Xinyu Che, Junqi Xiong, Chenchen Zhang, Yukai Huang, Chenyu Zhou, Haoyang Huang, Minghao Liu, Letian Zhu, Hongyi Ye, Ken Deng, Zizheng Zhan, Han Li, Yifan Yao, Ming Sun, Zhaoxiang Zhang, and Jiaheng Liu. 2026. WebCompass: Evaluating Multimodal Code Agents in Web Development from Vision, Instruction, and Interaction to Implementation. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 37948–37984, Budapest, Hungary. Association for Computational Linguistics.