@InProceedings{kilickaya-EtAl:2017:EACLlong,
  author    = {Kilickaya, Mert  and  Erdem, Aykut  and  Ikizler-Cinbis, Nazli  and  Erdem, Erkut},
  title     = {Re-evaluating Automatic Metrics for Image Captioning},
  booktitle = {Proceedings of the 15th Conference of the European Chapter of the Association for Computational Linguistics: Volume 1, Long Papers},
  month     = {April},
  year      = {2017},
  address   = {Valencia, Spain},
  publisher = {Association for Computational Linguistics},
  pages     = {199--209},
  abstract  = {The task of generating natural language descriptions from images has received a
	lot of attention in recent years. Consequently, it is becoming increasingly
	important to evaluate such image captioning approaches in an automatic manner.
	In this paper, we provide an in-depth evaluation of the existing image
	captioning metrics through a series of carefully designed experiments.
	Moreover, we explore the utilization of the recently proposed Word Mover's
	Distance (WMD) document metric for the purpose of image captioning. Our
	findings outline the differences and/or similarities between metrics and their
	relative robustness by means of extensive correlation, accuracy and distraction
	based evaluations. Our results also demonstrate that WMD provides strong
	advantages over other metrics.},
  url       = {http://www.aclweb.org/anthology/E17-1019}
}