@InProceedings{schofield-thompson-mimno:2017:EMNLP2017,
  author    = {Schofield, Alexandra  and  Thompson, Laure  and  Mimno, David},
  title     = {Quantifying the Effects of Text Duplication on Semantic Models},
  booktitle = {Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
  month     = {September},
  year      = {2017},
  address   = {Copenhagen, Denmark},
  publisher = {Association for Computational Linguistics},
  pages     = {2737--2747},
  abstract  = {Duplicate documents are a pervasive problem in text datasets and can have a
	strong effect on unsupervised models. Methods to remove duplicate texts are
	typically heuristic or very expensive, so it is vital to know when and why they
	are needed. We measure the sensitivity of two latent semantic methods to the
	presence of different levels of document repetition. By artificially creating
	different forms of duplicate text we confirm several hypotheses about how
	repeated text impacts models. While a small amount of duplication is tolerable,
	substantial over-representation of subsets of the text may overwhelm meaningful
	topical patterns.},
  url       = {https://www.aclweb.org/anthology/D17-1290}
}