@InProceedings{braun-EtAl:2017:W17-55,
  author    = {Braun, Daniel  and  Hernandez-Mendez, Adrian  and  Matthes, Florian  and  Langen, Manfred},
  title     = {Evaluating Natural Language Understanding Services for Conversational Question Answering Systems},
  booktitle = {Proceedings of the 18th Annual SIGdial Meeting on Discourse and Dialogue},
  month     = {August},
  year      = {2017},
  address   = {Saarbrücken, Germany},
  publisher = {Association for Computational Linguistics},
  pages     = {174--185},
  abstract  = {Conversational interfaces recently gained a lot of attention. One of the
	reasons for the current hype is the fact that chatbots (one particularly
	popular form of conversational interfaces)
	nowadays can be created without any programming knowledge, thanks to different
	toolkits and so-called Natural Language Understanding (NLU) services. While
	these NLU services are already widely used in both, industry and science, so
	far, they have not been analysed systematically. In this paper, we present a
	method to evaluate the classification performance of NLU services. Moreover, we
	present two new corpora, one consisting of annotated questions and one
	consisting of annotated questions with the corresponding answers. Based on
	these corpora, we conduct an evaluation of some of the most popular NLU
	services. Thereby we want to enable both, researchers and companies to make
	more educated decisions about which service they should use.},
  url       = {http://aclweb.org/anthology/W17-5522}
}