@inproceedings{kim-etal-2022-oh,
title = "Oh My Mistake!: Toward Realistic Dialogue State Tracking including Turnback Utterances",
author = "Kim, Takyoung and
Lee, Yukyung and
Yoon, Hoonsang and
Kang, Pilsung and
Bang, Junseong and
Kim, Misuk",
editor = "Ou, Zhijian and
Feng, Junlan and
Li, Juanzi",
booktitle = "Proceedings of the Towards Semi-Supervised and Reinforced Task-Oriented Dialog Systems (SereTOD)",
month = dec,
year = "2022",
address = "Abu Dhabi, Beijing (Hybrid)",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2022.seretod-1.1",
doi = "10.18653/v1/2022.seretod-1.1",
pages = "1--12",
abstract = "The primary purpose of dialogue state tracking(DST), a critical component of an end-toend conversational system, is to build a model that responds well to real-world situations. Although we often change our minds from time to time during ordinary conversations, current benchmark datasets do not adequately reflect such occurrences and instead consist of over-simplified conversations, in which no one changes their mind during a conversation. As the main question inspiring the present study, {``}Are current benchmark datasets sufficiently diverse to handle casual conversations in which one changes their mind after a certain topic is over?{''} We found that the answer is {``}No{''} because DST models cannot refer to previous user preferences when template-based turnback utterances are injected into the dataset. Even in the the simplest mind-changing (turnback) scenario, the performance of DST models significantly degenerated. However, we found that this performance degeneration can be recovered when the turnback scenarios are explicitly designed in the training set, implying that the problem is not with the DST models but rather with the construction of the benchmark dataset.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kim-etal-2022-oh">
<titleInfo>
<title>Oh My Mistake!: Toward Realistic Dialogue State Tracking including Turnback Utterances</title>
</titleInfo>
<name type="personal">
<namePart type="given">Takyoung</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yukyung</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hoonsang</namePart>
<namePart type="family">Yoon</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pilsung</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junseong</namePart>
<namePart type="family">Bang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Misuk</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2022-12</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Towards Semi-Supervised and Reinforced Task-Oriented Dialog Systems (SereTOD)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zhijian</namePart>
<namePart type="family">Ou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junlan</namePart>
<namePart type="family">Feng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Juanzi</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Abu Dhabi, Beijing (Hybrid)</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>The primary purpose of dialogue state tracking(DST), a critical component of an end-toend conversational system, is to build a model that responds well to real-world situations. Although we often change our minds from time to time during ordinary conversations, current benchmark datasets do not adequately reflect such occurrences and instead consist of over-simplified conversations, in which no one changes their mind during a conversation. As the main question inspiring the present study, “Are current benchmark datasets sufficiently diverse to handle casual conversations in which one changes their mind after a certain topic is over?” We found that the answer is “No” because DST models cannot refer to previous user preferences when template-based turnback utterances are injected into the dataset. Even in the the simplest mind-changing (turnback) scenario, the performance of DST models significantly degenerated. However, we found that this performance degeneration can be recovered when the turnback scenarios are explicitly designed in the training set, implying that the problem is not with the DST models but rather with the construction of the benchmark dataset.</abstract>
<identifier type="citekey">kim-etal-2022-oh</identifier>
<identifier type="doi">10.18653/v1/2022.seretod-1.1</identifier>
<location>
<url>https://aclanthology.org/2022.seretod-1.1</url>
</location>
<part>
<date>2022-12</date>
<extent unit="page">
<start>1</start>
<end>12</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Oh My Mistake!: Toward Realistic Dialogue State Tracking including Turnback Utterances
%A Kim, Takyoung
%A Lee, Yukyung
%A Yoon, Hoonsang
%A Kang, Pilsung
%A Bang, Junseong
%A Kim, Misuk
%Y Ou, Zhijian
%Y Feng, Junlan
%Y Li, Juanzi
%S Proceedings of the Towards Semi-Supervised and Reinforced Task-Oriented Dialog Systems (SereTOD)
%D 2022
%8 December
%I Association for Computational Linguistics
%C Abu Dhabi, Beijing (Hybrid)
%F kim-etal-2022-oh
%X The primary purpose of dialogue state tracking(DST), a critical component of an end-toend conversational system, is to build a model that responds well to real-world situations. Although we often change our minds from time to time during ordinary conversations, current benchmark datasets do not adequately reflect such occurrences and instead consist of over-simplified conversations, in which no one changes their mind during a conversation. As the main question inspiring the present study, “Are current benchmark datasets sufficiently diverse to handle casual conversations in which one changes their mind after a certain topic is over?” We found that the answer is “No” because DST models cannot refer to previous user preferences when template-based turnback utterances are injected into the dataset. Even in the the simplest mind-changing (turnback) scenario, the performance of DST models significantly degenerated. However, we found that this performance degeneration can be recovered when the turnback scenarios are explicitly designed in the training set, implying that the problem is not with the DST models but rather with the construction of the benchmark dataset.
%R 10.18653/v1/2022.seretod-1.1
%U https://aclanthology.org/2022.seretod-1.1
%U https://doi.org/10.18653/v1/2022.seretod-1.1
%P 1-12
Markdown (Informal)
[Oh My Mistake!: Toward Realistic Dialogue State Tracking including Turnback Utterances](https://aclanthology.org/2022.seretod-1.1) (Kim et al., SereTOD 2022)
ACL