@InProceedings{poliak-EtAl:2018:N18-2,
  author    = {Poliak, Adam  and  Belinkov, Yonatan  and  Glass, James  and  Van Durme, Benjamin},
  title     = {On the Evaluation of Semantic Phenomena in Neural Machine Translation Using Natural Language Inference},
  booktitle = {Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 2 (Short Papers)},
  month     = {June},
  year      = {2018},
  address   = {New Orleans, Louisiana},
  publisher = {Association for Computational Linguistics},
  pages     = {513--523},
  abstract  = {We propose a process for investigating the extent to which sentence representations arising from neural machine translation (NMT) systems encode distinct semantic phenomena. We use these representations as features to train a natural language inference (NLI) classifier based on datasets recast from existing semantic annotations. In applying this process to a representative NMT system, we find its encoder appears most suited to supporting inferences at the syntax-semantics interface, as compared to anaphora resolution requiring world knowledge. We conclude with a discussion on the merits and potential deficiencies of the existing process, and how it may be improved and extended as a broader framework for evaluating semantic coverage},
  url       = {http://www.aclweb.org/anthology/N18-2082}
}