@InProceedings{stanojevic-simaan:2016:COLING,
  author    = {Stanojevi\'{c}, Milo\v{s}  and  Sima'an, Khalil},
  title     = {Hierarchical Permutation Complexity for Word Order Evaluation},
  booktitle = {Proceedings of COLING 2016, the 26th International Conference on Computational Linguistics: Technical Papers},
  month     = {December},
  year      = {2016},
  address   = {Osaka, Japan},
  publisher = {The COLING 2016 Organizing Committee},
  pages     = {2164--2173},
  abstract  = {Existing approaches for evaluating word order in machine translation work with
	metrics computed directly over a permutation of word positions in system output
	relative to a reference translation. However, every permutation factorizes into
	a permutation tree (PET) built of primal permutations, i.e., atomic units that
	do not factorize any further. In this paper we explore the idea that
	permutations factorizing into (on average) shorter primal permutations should
	represent simpler ordering as well. Consequently, we contribute Permutation
	Complexity, a class of metrics over PETs and their extension to forests, and
	define tight metrics, a sub-class of metrics implementing this idea.
	Subsequently we define example tight metrics and empirically test them
	in word order evaluation. Experiments on the WMT13 data sets for ten language
	pairs show that a tight metric is more often than not better than the
	baselines.},
  url       = {http://aclweb.org/anthology/C16-1204}
}