@inproceedings{de-avalle-etal-2026-diagflowbench,
title = "{D}iag{F}low{B}ench: Evaluating How Language Models Handle Off-Procedure Inputs in Grounded Diagnostic Dialogue",
author = "de Avalle, Guillermo Gil and
Maruster, Laura and
Raza, Shaina and
Emmanouilidis, Christos",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.108/",
pages = "1854--1871",
ISBN = "979-8-89176-460-6",
abstract = "Language models increasingly serve as advisory systems in maintenance operations. To prevent hallucination, established approaches ground these models in procedural documentation, constraining them to prescribed sequences. In practice, however, operators may stray from these steps, requiring models to recognise unscripted, off-procedure utterances. Current benchmarks rarely prioritise this capability. We introduce DiagFlowBench, a dataset of 50 industrial diagnostic flowcharts from a consumer manufacturer converted into 1,676 multi-turn conversations that contrast compliant with off-procedure utterances. Evaluating a panel of ten commercial and open-weight models reveals high variability in abstention rates, with models often selecting a real but contextually inadequate step rather than fabricating facts. The inherent plausibility and authority of this mapped but wrong advice exposes a challenging vulnerability for grounding systems."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="de-avalle-etal-2026-diagflowbench">
<titleInfo>
<title>DiagFlowBench: Evaluating How Language Models Handle Off-Procedure Inputs in Grounded Diagnostic Dialogue</title>
</titleInfo>
<name type="personal">
<namePart type="given">Guillermo</namePart>
<namePart type="given">Gil</namePart>
<namePart type="family">de Avalle</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Laura</namePart>
<namePart type="family">Maruster</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shaina</namePart>
<namePart type="family">Raza</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christos</namePart>
<namePart type="family">Emmanouilidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>Language models increasingly serve as advisory systems in maintenance operations. To prevent hallucination, established approaches ground these models in procedural documentation, constraining them to prescribed sequences. In practice, however, operators may stray from these steps, requiring models to recognise unscripted, off-procedure utterances. Current benchmarks rarely prioritise this capability. We introduce DiagFlowBench, a dataset of 50 industrial diagnostic flowcharts from a consumer manufacturer converted into 1,676 multi-turn conversations that contrast compliant with off-procedure utterances. Evaluating a panel of ten commercial and open-weight models reveals high variability in abstention rates, with models often selecting a real but contextually inadequate step rather than fabricating facts. The inherent plausibility and authority of this mapped but wrong advice exposes a challenging vulnerability for grounding systems.</abstract>
<identifier type="citekey">de-avalle-etal-2026-diagflowbench</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.108/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1854</start>
<end>1871</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T DiagFlowBench: Evaluating How Language Models Handle Off-Procedure Inputs in Grounded Diagnostic Dialogue
%A de Avalle, Guillermo Gil
%A Maruster, Laura
%A Raza, Shaina
%A Emmanouilidis, Christos
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F de-avalle-etal-2026-diagflowbench
%X Language models increasingly serve as advisory systems in maintenance operations. To prevent hallucination, established approaches ground these models in procedural documentation, constraining them to prescribed sequences. In practice, however, operators may stray from these steps, requiring models to recognise unscripted, off-procedure utterances. Current benchmarks rarely prioritise this capability. We introduce DiagFlowBench, a dataset of 50 industrial diagnostic flowcharts from a consumer manufacturer converted into 1,676 multi-turn conversations that contrast compliant with off-procedure utterances. Evaluating a panel of ten commercial and open-weight models reveals high variability in abstention rates, with models often selecting a real but contextually inadequate step rather than fabricating facts. The inherent plausibility and authority of this mapped but wrong advice exposes a challenging vulnerability for grounding systems.
%U https://aclanthology.org/2026.emnlp-industry.108/
%P 1854-1871
Markdown (Informal)
[DiagFlowBench: Evaluating How Language Models Handle Off-Procedure Inputs in Grounded Diagnostic Dialogue](https://aclanthology.org/2026.emnlp-industry.108/) (de Avalle et al., EMNLP 2026)
ACL