@inproceedings{chambers-2017-behind,
title = "Behind the Scenes of an Evolving Event Cloze Test",
author = "Chambers, Nathanael",
editor = "Roth, Michael and
Mostafazadeh, Nasrin and
Chambers, Nathanael and
Louis, Annie",
booktitle = "Proceedings of the 2nd Workshop on Linking Models of Lexical, Sentential and Discourse-level Semantics",
month = apr,
year = "2017",
address = "Valencia, Spain",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/W17-0905",
doi = "10.18653/v1/W17-0905",
pages = "41--45",
abstract = "This paper analyzes the narrative event cloze test and its recent evolution. The test removes one event from a document{'}s chain of events, and systems predict the missing event. Originally proposed to evaluate learned knowledge of event scenarios (e.g., scripts and frames), most recent work now builds ngram-like language models (LM) to beat the test. This paper argues that the test has slowly/unknowingly been altered to accommodate LMs.5 Most notably, tests are auto-generated rather than by hand, and no effort is taken to include core script events. Recent work is not clear on evaluation goals and contains contradictory results. We implement several models, and show that the test{'}s bias to high-frequency events explains the inconsistencies. We conclude with recommendations on how to return to the test{'}s original intent, and offer brief suggestions on a path forward.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chambers-2017-behind">
<titleInfo>
<title>Behind the Scenes of an Evolving Event Cloze Test</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nathanael</namePart>
<namePart type="family">Chambers</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2017-04</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2nd Workshop on Linking Models of Lexical, Sentential and Discourse-level Semantics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Michael</namePart>
<namePart type="family">Roth</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nasrin</namePart>
<namePart type="family">Mostafazadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nathanael</namePart>
<namePart type="family">Chambers</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Annie</namePart>
<namePart type="family">Louis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Valencia, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>This paper analyzes the narrative event cloze test and its recent evolution. The test removes one event from a document’s chain of events, and systems predict the missing event. Originally proposed to evaluate learned knowledge of event scenarios (e.g., scripts and frames), most recent work now builds ngram-like language models (LM) to beat the test. This paper argues that the test has slowly/unknowingly been altered to accommodate LMs.5 Most notably, tests are auto-generated rather than by hand, and no effort is taken to include core script events. Recent work is not clear on evaluation goals and contains contradictory results. We implement several models, and show that the test’s bias to high-frequency events explains the inconsistencies. We conclude with recommendations on how to return to the test’s original intent, and offer brief suggestions on a path forward.</abstract>
<identifier type="citekey">chambers-2017-behind</identifier>
<identifier type="doi">10.18653/v1/W17-0905</identifier>
<location>
<url>https://aclanthology.org/W17-0905</url>
</location>
<part>
<date>2017-04</date>
<extent unit="page">
<start>41</start>
<end>45</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Behind the Scenes of an Evolving Event Cloze Test
%A Chambers, Nathanael
%Y Roth, Michael
%Y Mostafazadeh, Nasrin
%Y Chambers, Nathanael
%Y Louis, Annie
%S Proceedings of the 2nd Workshop on Linking Models of Lexical, Sentential and Discourse-level Semantics
%D 2017
%8 April
%I Association for Computational Linguistics
%C Valencia, Spain
%F chambers-2017-behind
%X This paper analyzes the narrative event cloze test and its recent evolution. The test removes one event from a document’s chain of events, and systems predict the missing event. Originally proposed to evaluate learned knowledge of event scenarios (e.g., scripts and frames), most recent work now builds ngram-like language models (LM) to beat the test. This paper argues that the test has slowly/unknowingly been altered to accommodate LMs.5 Most notably, tests are auto-generated rather than by hand, and no effort is taken to include core script events. Recent work is not clear on evaluation goals and contains contradictory results. We implement several models, and show that the test’s bias to high-frequency events explains the inconsistencies. We conclude with recommendations on how to return to the test’s original intent, and offer brief suggestions on a path forward.
%R 10.18653/v1/W17-0905
%U https://aclanthology.org/W17-0905
%U https://doi.org/10.18653/v1/W17-0905
%P 41-45
Markdown (Informal)
[Behind the Scenes of an Evolving Event Cloze Test](https://aclanthology.org/W17-0905) (Chambers, LSDSem 2017)
ACL
- Nathanael Chambers. 2017. Behind the Scenes of an Evolving Event Cloze Test. In Proceedings of the 2nd Workshop on Linking Models of Lexical, Sentential and Discourse-level Semantics, pages 41–45, Valencia, Spain. Association for Computational Linguistics.