@InProceedings{vulic-kiela-korhonen:2017:EACLlong,
  author    = {Vuli\'{c}, Ivan  and  Kiela, Douwe  and  Korhonen, Anna},
  title     = {Evaluation by Association: A Systematic Study of Quantitative Word Association Evaluation},
  booktitle = {Proceedings of the 15th Conference of the European Chapter of the Association for Computational Linguistics: Volume 1, Long Papers},
  month     = {April},
  year      = {2017},
  address   = {Valencia, Spain},
  publisher = {Association for Computational Linguistics},
  pages     = {163--175},
  abstract  = {Recent work on evaluating representation learning architectures in NLP has
	established a need for evaluation protocols based on subconscious cognitive
	measures rather than manually tailored intrinsic similarity and relatedness
	tasks. In this work, we propose a novel evaluation framework that enables
	large-scale evaluation of such architectures in the free word association (WA)
	task, which is firmly grounded in cognitive theories of human semantic
	representation. This evaluation is facilitated by the existence of large
	manually constructed repositories of word association data. In this paper, we
	(1) present a detailed analysis of the new quantitative WA evaluation protocol,
	(2) suggest new evaluation metrics for the WA task inspired by its direct
	analogy with information retrieval problems, (3) evaluate various
	state-of-the-art representation models on this task, and (4) discuss the
	relationship between WA and prior evaluations of semantic representation with
	well-known similarity and relatedness evaluation sets. We have made the WA
	evaluation toolkit publicly available.},
  url       = {http://www.aclweb.org/anthology/E17-1016}
}