@InProceedings{ngongangomo-EtAl:2018:W18-65,
  author    = {Ngonga Ngomo, Axel-Cyrille  and  Röder, Michael  and  Moussallem, Diego  and  Usbeck, Ricardo  and  Speck, René},
  title     = {BENGAL: An Automatic Benchmark Generator for Entity Recognition and Linking},
  booktitle = {Proceedings of the 11th International Conference on Natural Language Generation},
  month     = {September},
  year      = {2018},
  address   = {Tilburg University, The Netherlands},
  publisher = {Association for Computational Linguistics},
  pages     = {339--349},
  abstract  = {The manual creation of gold standards for named entity recognition and entity linking is time- and resource-intensive. Moreover, recent works show that such gold standards contain a large proportion of mistakes in addition to being difficult to maintain. We hence present \Bengal{}, a novel automatic generation of such gold standards as a complement to manually created benchmarks. The main advantage of our benchmarks is that they can be readily generated at any time, and are cost-effective while being guaranteed to be free of annotation errors. We compare the performance of 11 tools on benchmarks in English generated by \Bengal{} with their performance on 16 benchmarks created manually. We show that our approach can be ported easily across languages by presenting results achieved on Brazilian Portuguese and Spanish. Overall, our results suggest that our automatic benchmark generation approach can create varied benchmarks that have characteristics similar to those of existing benchmarks. Our approach is open-source. Our experimental results are available at~\url{  url       = {http://www.aclweb.org/anthology/W18-65, http://www.aclweb.org/anthology/W18-6541}
}