@InProceedings{thomason-sinapov-mooney:2017:RoboNLP,
  author    = {Thomason, Jesse  and  Sinapov, Jivko  and  Mooney, Raymond},
  title     = {Guiding Interaction Behaviors for Multi-modal Grounded Language Learning},
  booktitle = {Proceedings of the First Workshop on Language Grounding for Robotics},
  month     = {August},
  year      = {2017},
  address   = {Vancouver, Canada},
  publisher = {Association for Computational Linguistics},
  pages     = {20--24},
  abstract  = {Multi-modal grounded language learning connects language predicates to physical
	properties of objects in the world. Sensing with multiple modalities, such as
	audio, haptics, and visual colors and shapes while performing interaction
	behaviors like lifting, dropping, and looking on objects enables a robot to
	ground non-visual predicates like ``empty'' as well as visual predicates like
	``red''. Previous work has established that grounding in multi-modal space
	improves performance on object retrieval from human descriptions. In this work,
	we gather behavior annotations from humans and demonstrate that these improve
	language grounding performance by allowing a system to focus on relevant
	behaviors for words like ``white'' or ``half-full'' that can be understood by
	looking or lifting, respectively. We also explore adding modality annotations
	(whether to focus on audio or haptics when performing a behavior), which
	improves performance, and sharing information between linguistically related
	predicates (if ``green'' is a color, ``white'' is a color), which improves
	grounding recall but at the cost of precision.},
  url       = {http://www.aclweb.org/anthology/W17-2803}
}