@inproceedings{kuksenok-martyniv-2019-evaluation,
title = "Evaluation and Improvement of Chatbot Text Classification Data Quality Using Plausible Negative Examples",
author = "Kuksenok, Kit and
Martyniv, Andriy",
editor = "Chen, Yun-Nung and
Bedrax-Weiss, Tania and
Hakkani-Tur, Dilek and
Kumar, Anuj and
Lewis, Mike and
Luong, Thang-Minh and
Su, Pei-Hao and
Wen, Tsung-Hsien",
booktitle = "Proceedings of the First Workshop on NLP for Conversational AI",
month = aug,
year = "2019",
address = "Florence, Italy",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/W19-4110/",
doi = "10.18653/v1/W19-4110",
pages = "87--95",
abstract = "We describe and validate a metric for estimating multi-class classifier performance based on cross-validation and adapted for improvement of small, unbalanced natural-language datasets used in chatbot design. Our experiences draw upon building recruitment chatbots that mediate communication between job-seekers and recruiters by exposing the ML/NLP dataset to the recruiting team. Evaluation approaches must be understandable to various stakeholders, and useful for improving chatbot performance. The metric, nex-cv, uses negative examples in the evaluation of text classification, and fulfils three requirements. First, it is actionable: it can be used by non-developer staff. Second, it is not overly optimistic compared to human ratings, making it a fast method for comparing classifiers. Third, it allows model-agnostic comparison, making it useful for comparing systems despite implementation differences. We validate the metric based on seven recruitment-domain datasets in English and German over the course of one year."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kuksenok-martyniv-2019-evaluation">
<titleInfo>
<title>Evaluation and Improvement of Chatbot Text Classification Data Quality Using Plausible Negative Examples</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kit</namePart>
<namePart type="family">Kuksenok</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andriy</namePart>
<namePart type="family">Martyniv</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2019-08</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on NLP for Conversational AI</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yun-Nung</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tania</namePart>
<namePart type="family">Bedrax-Weiss</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dilek</namePart>
<namePart type="family">Hakkani-Tur</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anuj</namePart>
<namePart type="family">Kumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mike</namePart>
<namePart type="family">Lewis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thang-Minh</namePart>
<namePart type="family">Luong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pei-Hao</namePart>
<namePart type="family">Su</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tsung-Hsien</namePart>
<namePart type="family">Wen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Florence, Italy</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>We describe and validate a metric for estimating multi-class classifier performance based on cross-validation and adapted for improvement of small, unbalanced natural-language datasets used in chatbot design. Our experiences draw upon building recruitment chatbots that mediate communication between job-seekers and recruiters by exposing the ML/NLP dataset to the recruiting team. Evaluation approaches must be understandable to various stakeholders, and useful for improving chatbot performance. The metric, nex-cv, uses negative examples in the evaluation of text classification, and fulfils three requirements. First, it is actionable: it can be used by non-developer staff. Second, it is not overly optimistic compared to human ratings, making it a fast method for comparing classifiers. Third, it allows model-agnostic comparison, making it useful for comparing systems despite implementation differences. We validate the metric based on seven recruitment-domain datasets in English and German over the course of one year.</abstract>
<identifier type="citekey">kuksenok-martyniv-2019-evaluation</identifier>
<identifier type="doi">10.18653/v1/W19-4110</identifier>
<location>
<url>https://aclanthology.org/W19-4110/</url>
</location>
<part>
<date>2019-08</date>
<extent unit="page">
<start>87</start>
<end>95</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluation and Improvement of Chatbot Text Classification Data Quality Using Plausible Negative Examples
%A Kuksenok, Kit
%A Martyniv, Andriy
%Y Chen, Yun-Nung
%Y Bedrax-Weiss, Tania
%Y Hakkani-Tur, Dilek
%Y Kumar, Anuj
%Y Lewis, Mike
%Y Luong, Thang-Minh
%Y Su, Pei-Hao
%Y Wen, Tsung-Hsien
%S Proceedings of the First Workshop on NLP for Conversational AI
%D 2019
%8 August
%I Association for Computational Linguistics
%C Florence, Italy
%F kuksenok-martyniv-2019-evaluation
%X We describe and validate a metric for estimating multi-class classifier performance based on cross-validation and adapted for improvement of small, unbalanced natural-language datasets used in chatbot design. Our experiences draw upon building recruitment chatbots that mediate communication between job-seekers and recruiters by exposing the ML/NLP dataset to the recruiting team. Evaluation approaches must be understandable to various stakeholders, and useful for improving chatbot performance. The metric, nex-cv, uses negative examples in the evaluation of text classification, and fulfils three requirements. First, it is actionable: it can be used by non-developer staff. Second, it is not overly optimistic compared to human ratings, making it a fast method for comparing classifiers. Third, it allows model-agnostic comparison, making it useful for comparing systems despite implementation differences. We validate the metric based on seven recruitment-domain datasets in English and German over the course of one year.
%R 10.18653/v1/W19-4110
%U https://aclanthology.org/W19-4110/
%U https://doi.org/10.18653/v1/W19-4110
%P 87-95
Markdown (Informal)
[Evaluation and Improvement of Chatbot Text Classification Data Quality Using Plausible Negative Examples](https://aclanthology.org/W19-4110/) (Kuksenok & Martyniv, ACL 2019)
ACL