@InProceedings{upadhyay-EtAl:2016:COLING,
  author    = {Upadhyay, Shyam  and  Gupta, Nitish  and  Christodoulopoulos, Christos  and  Roth, Dan},
  title     = {Revisiting the Evaluation for Cross Document Event Coreference},
  booktitle = {Proceedings of COLING 2016, the 26th International Conference on Computational Linguistics: Technical Papers},
  month     = {December},
  year      = {2016},
  address   = {Osaka, Japan},
  publisher = {The COLING 2016 Organizing Committee},
  pages     = {1949--1958},
  abstract  = {Cross document event coreference (CDEC) is an important task that aims at
	aggregating event- related information across multiple documents. We revisit
	the evaluation for CDEC, and dis- cover that past works have adopted different,
	often inconsistent, evaluation settings, which either overlook certain mistakes
	in coreference decisions, or make assumptions that simplify the coreference
	task considerably. We suggest a new evaluation methodology which overcomes
	these limitations, and allows for an accurate assessment of CDEC systems. Our
	new evaluation setting better reflects the corpus-wide information aggregation
	ability of CDEC systems by separating event-coreference decisions made across
	documents from those made within a document. In addition, we suggest a better
	baseline for the task and semi-automatically identify several inconsistent
	annotations in the evaluation dataset.},
  url       = {http://aclweb.org/anthology/C16-1183}
}