@inproceedings{zhang-etal-2026-querydiagrameval,
title = "{Q}uery{D}iagram{E}val: Evaluating and Benchmarking {LLM} Diagram Generation under Noisy Conditions",
author = "Zhang, Ted and
Das, Souvik and
Ni, Kai and
Agrawal, Aditya and
Guo, Xin and
Kochedykov, Denis",
editor = "Bak, JinYeong and
van der Goot, Rob and
Jang, Hyeju and
Buaphet, Weerayut and
Ramponi, Alan and
Xu, Wei and
Ritter, Alan",
booktitle = "Proceedings of the 11th Workshop on Natural User-generated Text",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.wnut-1.18/",
pages = "244--265",
ISBN = "979-8-89176-470-5",
abstract = "Chat assistants increasingly ask large language models to turn free-form user prose into structured, executable diagrams. These requests are noisy user-generated text: informal, hedged, jargon-mixed, incomplete, and often describing the target only indirectly{---}in workflow language rather than diagram vocabulary. Evaluation is hard because many distinct diagrams satisfy the same noisy request, making reference-based metrics too rigid where no ground truth exists, while reference-free LLM judges give weak, hard-to-reproduce diagnostics. We introduce \textsc{QueryDiagramEval} (QDE), a reference-free protocol scoring diagram generation directly against raw, noisy queries via two levels: deterministic compilation gates (syntax, rendering, diagram-type correctness), then a semantic-fidelity layer where a multimodal auditor is grounded in type-specific structural facts parsed from the code. On 255 annotated predictions, QDE aligns strongly with holistic human ratings ($\rho=0.715$), outperforming an ungrounded LLM judge. We also release \textsc{MermaidBench}, a source-grounded benchmark of 1,150 query{--}gold pairs over 11 diagram types, four query-difficulty tiers, and a multi-turn track, with the gold held fixed to isolate robustness to input-side noise. Furthermore, we conduct experiments on eight frontier-model and harnesses to show that the our agentic framework reaches near-gold quality while failures concentrate in topology preservation, label grounding, and type-specific syntax."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-querydiagrameval">
<titleInfo>
<title>QueryDiagramEval: Evaluating and Benchmarking LLM Diagram Generation under Noisy Conditions</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ted</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Souvik</namePart>
<namePart type="family">Das</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kai</namePart>
<namePart type="family">Ni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aditya</namePart>
<namePart type="family">Agrawal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Guo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Denis</namePart>
<namePart type="family">Kochedykov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 11th Workshop on Natural User-generated Text</title>
</titleInfo>
<name type="personal">
<namePart type="given">JinYeong</namePart>
<namePart type="family">Bak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rob</namePart>
<namePart type="family">van der Goot</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hyeju</namePart>
<namePart type="family">Jang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weerayut</namePart>
<namePart type="family">Buaphet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Ramponi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Ritter</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-470-5</identifier>
</relatedItem>
<abstract>Chat assistants increasingly ask large language models to turn free-form user prose into structured, executable diagrams. These requests are noisy user-generated text: informal, hedged, jargon-mixed, incomplete, and often describing the target only indirectly—in workflow language rather than diagram vocabulary. Evaluation is hard because many distinct diagrams satisfy the same noisy request, making reference-based metrics too rigid where no ground truth exists, while reference-free LLM judges give weak, hard-to-reproduce diagnostics. We introduce QueryDiagramEval (QDE), a reference-free protocol scoring diagram generation directly against raw, noisy queries via two levels: deterministic compilation gates (syntax, rendering, diagram-type correctness), then a semantic-fidelity layer where a multimodal auditor is grounded in type-specific structural facts parsed from the code. On 255 annotated predictions, QDE aligns strongly with holistic human ratings (ρ=0.715), outperforming an ungrounded LLM judge. We also release MermaidBench, a source-grounded benchmark of 1,150 query–gold pairs over 11 diagram types, four query-difficulty tiers, and a multi-turn track, with the gold held fixed to isolate robustness to input-side noise. Furthermore, we conduct experiments on eight frontier-model and harnesses to show that the our agentic framework reaches near-gold quality while failures concentrate in topology preservation, label grounding, and type-specific syntax.</abstract>
<identifier type="citekey">zhang-etal-2026-querydiagrameval</identifier>
<location>
<url>https://aclanthology.org/2026.wnut-1.18/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>244</start>
<end>265</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T QueryDiagramEval: Evaluating and Benchmarking LLM Diagram Generation under Noisy Conditions
%A Zhang, Ted
%A Das, Souvik
%A Ni, Kai
%A Agrawal, Aditya
%A Guo, Xin
%A Kochedykov, Denis
%Y Bak, JinYeong
%Y van der Goot, Rob
%Y Jang, Hyeju
%Y Buaphet, Weerayut
%Y Ramponi, Alan
%Y Xu, Wei
%Y Ritter, Alan
%S Proceedings of the 11th Workshop on Natural User-generated Text
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-470-5
%F zhang-etal-2026-querydiagrameval
%X Chat assistants increasingly ask large language models to turn free-form user prose into structured, executable diagrams. These requests are noisy user-generated text: informal, hedged, jargon-mixed, incomplete, and often describing the target only indirectly—in workflow language rather than diagram vocabulary. Evaluation is hard because many distinct diagrams satisfy the same noisy request, making reference-based metrics too rigid where no ground truth exists, while reference-free LLM judges give weak, hard-to-reproduce diagnostics. We introduce QueryDiagramEval (QDE), a reference-free protocol scoring diagram generation directly against raw, noisy queries via two levels: deterministic compilation gates (syntax, rendering, diagram-type correctness), then a semantic-fidelity layer where a multimodal auditor is grounded in type-specific structural facts parsed from the code. On 255 annotated predictions, QDE aligns strongly with holistic human ratings (ρ=0.715), outperforming an ungrounded LLM judge. We also release MermaidBench, a source-grounded benchmark of 1,150 query–gold pairs over 11 diagram types, four query-difficulty tiers, and a multi-turn track, with the gold held fixed to isolate robustness to input-side noise. Furthermore, we conduct experiments on eight frontier-model and harnesses to show that the our agentic framework reaches near-gold quality while failures concentrate in topology preservation, label grounding, and type-specific syntax.
%U https://aclanthology.org/2026.wnut-1.18/
%P 244-265
Markdown (Informal)
[QueryDiagramEval: Evaluating and Benchmarking LLM Diagram Generation under Noisy Conditions](https://aclanthology.org/2026.wnut-1.18/) (Zhang et al., WNUT 2026)
ACL