@inproceedings{hsu-etal-2026-evaluating,
title = "Evaluating {LLM}-generated Explanatory Dialogue Turns through Dialogue Completion",
author = "Hsu, Yi-Sheng and
Feldhus, Nils and
Stede, Manfred",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.72/",
pages = "1649--1679",
ISBN = "979-8-89176-459-0",
abstract = "Synthetic explanatory dialogues hold promise for applications from tutoring systems to model rationalization, yet their quality is often deemed inferior to human-authored conversations. This study uses dialogue completion as a lens to investigate the nuanced differences between human-spoken, human-written, and synthetic explanatory dialogue. By evaluating LLM-generated turns against original turns using the FED and IXQuisite test suites, we uncover a significant bias in the FED metric towards written language, which may penalize the interactivity of spoken dialogue. We find that three popular LLMs (Mistral, Llama, Claude) replicate this pattern, excelling in fluency, but struggling with conversational engagement, particularly on spoken dialogue transcripts. More importantly, we demonstrate that these models can significantly improve their performance through a zero-shot, iterative self-refinement method. By providing feedback based on their specific weaknesses, we show that LLMs can iteratively adapt their outputs to better resemble the interactive qualities of human dialogue."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hsu-etal-2026-evaluating">
<titleInfo>
<title>Evaluating LLM-generated Explanatory Dialogue Turns through Dialogue Completion</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yi-Sheng</namePart>
<namePart type="family">Hsu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nils</namePart>
<namePart type="family">Feldhus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Manfred</namePart>
<namePart type="family">Stede</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Synthetic explanatory dialogues hold promise for applications from tutoring systems to model rationalization, yet their quality is often deemed inferior to human-authored conversations. This study uses dialogue completion as a lens to investigate the nuanced differences between human-spoken, human-written, and synthetic explanatory dialogue. By evaluating LLM-generated turns against original turns using the FED and IXQuisite test suites, we uncover a significant bias in the FED metric towards written language, which may penalize the interactivity of spoken dialogue. We find that three popular LLMs (Mistral, Llama, Claude) replicate this pattern, excelling in fluency, but struggling with conversational engagement, particularly on spoken dialogue transcripts. More importantly, we demonstrate that these models can significantly improve their performance through a zero-shot, iterative self-refinement method. By providing feedback based on their specific weaknesses, we show that LLMs can iteratively adapt their outputs to better resemble the interactive qualities of human dialogue.</abstract>
<identifier type="citekey">hsu-etal-2026-evaluating</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.72/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1649</start>
<end>1679</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating LLM-generated Explanatory Dialogue Turns through Dialogue Completion
%A Hsu, Yi-Sheng
%A Feldhus, Nils
%A Stede, Manfred
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F hsu-etal-2026-evaluating
%X Synthetic explanatory dialogues hold promise for applications from tutoring systems to model rationalization, yet their quality is often deemed inferior to human-authored conversations. This study uses dialogue completion as a lens to investigate the nuanced differences between human-spoken, human-written, and synthetic explanatory dialogue. By evaluating LLM-generated turns against original turns using the FED and IXQuisite test suites, we uncover a significant bias in the FED metric towards written language, which may penalize the interactivity of spoken dialogue. We find that three popular LLMs (Mistral, Llama, Claude) replicate this pattern, excelling in fluency, but struggling with conversational engagement, particularly on spoken dialogue transcripts. More importantly, we demonstrate that these models can significantly improve their performance through a zero-shot, iterative self-refinement method. By providing feedback based on their specific weaknesses, we show that LLMs can iteratively adapt their outputs to better resemble the interactive qualities of human dialogue.
%U https://aclanthology.org/2026.emnlp-main.72/
%P 1649-1679
Markdown (Informal)
[Evaluating LLM-generated Explanatory Dialogue Turns through Dialogue Completion](https://aclanthology.org/2026.emnlp-main.72/) (Hsu et al., EMNLP 2026)
ACL