@InProceedings{isabelle-cherry-foster:2017:EMNLP2017,
  author    = {Isabelle, Pierre  and  Cherry, Colin  and  Foster, George},
  title     = {A Challenge Set Approach to Evaluating Machine Translation},
  booktitle = {Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
  month     = {September},
  year      = {2017},
  address   = {Copenhagen, Denmark},
  publisher = {Association for Computational Linguistics},
  pages     = {2486--2496},
  abstract  = {Neural machine translation represents an exciting leap forward in translation
	quality. But what longstanding weaknesses does it resolve, and which remain?
	We address these questions with a challenge set approach to translation
	evaluation and error analysis. A challenge set consists of a small set of
	sentences, each hand-designed to probe a system's capacity to bridge a
	particular structural divergence between languages.  To exemplify this
	approach, we present an English-French challenge set, and use it to analyze
	phrase-based and neural systems. The resulting analysis provides not only a
	more fine-grained picture of the strengths of neural systems, but also insight
	into which linguistic phenomena remain out of reach.},
  url       = {https://www.aclweb.org/anthology/D17-1263}
}