@InProceedings{parde-nielsen:2017:EMNLP2017,
  author    = {Parde, Natalie  and  Nielsen, Rodney},
  title     = {Finding Patterns in Noisy Crowds: Regression-based Annotation Aggregation for Crowdsourced Data},
  booktitle = {Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
  month     = {September},
  year      = {2017},
  address   = {Copenhagen, Denmark},
  publisher = {Association for Computational Linguistics},
  pages     = {1907--1912},
  abstract  = {Crowdsourcing offers a convenient means of obtaining labeled data quickly and
	inexpensively.              However, crowdsourced labels are often noisier than
	expert-annotated data, making it difficult to aggregate them meaningfully.  We
	present an aggregation approach that learns a regression model from
	crowdsourced annotations to predict aggregated labels for instances that have
	no expert adjudications.  The predicted labels achieve a correlation of 0.594
	with expert labels on our data, outperforming the best alternative aggregation
	method by 11.9%.  Our approach also outperforms the alternatives on third-party
	datasets.},
  url       = {https://www.aclweb.org/anthology/D17-1204}
}