@InProceedings{mehrabani-thomson-stern:2018:N18-3,
  author    = {Mehrabani, Mahnoosh  and  Thomson, David  and  Stern, Benjamin},
  title     = {Practical Application of Domain Dependent Confidence Measurement for Spoken Language Understanding Systems},
  booktitle = {Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 3 (Industry Papers)},
  month     = {June},
  year      = {2018},
  address   = {New Orleans - Louisiana},
  publisher = {Association for Computational Linguistics},
  pages     = {185--192},
  abstract  = {Spoken Language Understanding (SLU), which extracts semantic information from speech, is not flawless, specially in practical applications. The reliability of the output of an SLU system can be evaluated using a semantic confidence measure. Confidence measures are a solution to improve the quality of spoken dialogue systems, by rejecting low-confidence SLU results. In this study we discuss real-world applications of confidence scoring in a customer service scenario. We build confidence models for three major types of dialogue states that are considered as different domains: how may I help you, number capture, and confirmation. Practical challenges to train domain-dependent confidence models, including data limitations, are discussed, and it is shown that feature engineering plays an important role to improve performance. We explore a wide variety of predictor features based on speech recognition, intent classification, and high-level domain knowledge, and find the combined feature set with the best rejection performance for each application.},
  url       = {http://www.aclweb.org/anthology/N18-3023}
}