@InProceedings{nguyen-EtAl:2017:Long,
  author    = {Nguyen, An Thanh  and  Wallace, Byron  and  Li, Junyi Jessy  and  Nenkova, Ani  and  Lease, Matthew},
  title     = {Aggregating and Predicting Sequence Labels from Crowd Annotations},
  booktitle = {Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  month     = {July},
  year      = {2017},
  address   = {Vancouver, Canada},
  publisher = {Association for Computational Linguistics},
  pages     = {299--309},
  abstract  = {Despite sequences being core to NLP, scant work has considered how to handle
	noisy sequence labels from multiple annotators for the same text. Given such
	annotations, we consider two complementary tasks:  (1) aggregating sequential
	crowd labels to infer a best single set of consensus annotations; and (2) using
	crowd annotations as training data for a model that can predict sequences in
	unannotated text. For aggregation, we propose a novel Hidden Markov Model
	variant. To predict sequences in unannotated text, we propose a neural approach
	using Long Short Term Memory. We evaluate a suite of methods across two
	different applications and text genres: Named-Entity Recognition in news
	articles and Information Extraction from biomedical abstracts. Results show
	improvement over strong baselines. Our source code and data are available
	online.},
  url       = {http://aclweb.org/anthology/P17-1028}
}