@InProceedings{jansen-EtAl:2016:COLING,
  author    = {Jansen, Peter  and  Balasubramanian, Niranjan  and  Surdeanu, Mihai  and  Clark, Peter},
  title     = {What's in an Explanation? Characterizing Knowledge and Inference Requirements for Elementary Science Exams},
  booktitle = {Proceedings of COLING 2016, the 26th International Conference on Computational Linguistics: Technical Papers},
  month     = {December},
  year      = {2016},
  address   = {Osaka, Japan},
  publisher = {The COLING 2016 Organizing Committee},
  pages     = {2956--2965},
  abstract  = {QA systems have been making steady advances in the challenging elementary
	science exam domain. In this work, we develop an explanation-based analysis of
	knowledge and inference requirements, which supports a fine-grained
	characterization of the challenges. In particular, we model the requirements
	based on appropriate sources of evidence to be used for the QA task. We create
	requirements by first identifying suitable sentences in a knowledge base that
	support the correct answer, then use these to build explanations, filling in
	any necessary missing information. These explanations are used to create a
	fine-grained categorization of the requirements. Using these requirements, we
	compare a retrieval and an inference solver on 212 questions. The analysis
	validates the gains of the inference solver, demonstrating that it answers more
	questions requiring complex inference, while also providing insights into the
	relative strengths of the solvers and knowledge sources. We release the
	annotated questions and explanations as a resource with broad utility for
	science exam QA, including determining knowledge base construction targets, as
	well as supporting information aggregation in automated inference.},
  url       = {http://aclweb.org/anthology/C16-1278}
}