@inproceedings{deng-etal-2022-model,
title = "Model Criticism for Long-Form Text Generation",
author = "Deng, Yuntian and
Kuleshov, Volodymyr and
Rush, Alexander",
editor = "Goldberg, Yoav and
Kozareva, Zornitsa and
Zhang, Yue",
booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",
month = dec,
year = "2022",
address = "Abu Dhabi, United Arab Emirates",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2022.emnlp-main.815",
doi = "10.18653/v1/2022.emnlp-main.815",
pages = "11887--11912",
abstract = "Language models have demonstrated the ability to generate highly fluent text; however, it remains unclear whether their output retains coherent high-level structure (e.g., story progression). Here, we propose to apply a statistical tool, model criticism in latent space, to evaluate the high-level structure of the generated text. Model criticism compares the distributions between real and generated data in a latent space obtained according to an assumptive generative process. Different generative processes identify specific failure modes of the underlying model. We perform experiments on three representative aspects of high-level discourse{---}coherence, coreference, and topicality{---}and find that transformer-based language models are able to capture topical structures but have a harder time maintaining structural coherence or modeling coreference.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="deng-etal-2022-model">
<titleInfo>
<title>Model Criticism for Long-Form Text Generation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yuntian</namePart>
<namePart type="family">Deng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Volodymyr</namePart>
<namePart type="family">Kuleshov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexander</namePart>
<namePart type="family">Rush</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2022-12</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yoav</namePart>
<namePart type="family">Goldberg</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zornitsa</namePart>
<namePart type="family">Kozareva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Abu Dhabi, United Arab Emirates</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Language models have demonstrated the ability to generate highly fluent text; however, it remains unclear whether their output retains coherent high-level structure (e.g., story progression). Here, we propose to apply a statistical tool, model criticism in latent space, to evaluate the high-level structure of the generated text. Model criticism compares the distributions between real and generated data in a latent space obtained according to an assumptive generative process. Different generative processes identify specific failure modes of the underlying model. We perform experiments on three representative aspects of high-level discourse—coherence, coreference, and topicality—and find that transformer-based language models are able to capture topical structures but have a harder time maintaining structural coherence or modeling coreference.</abstract>
<identifier type="citekey">deng-etal-2022-model</identifier>
<identifier type="doi">10.18653/v1/2022.emnlp-main.815</identifier>
<location>
<url>https://aclanthology.org/2022.emnlp-main.815</url>
</location>
<part>
<date>2022-12</date>
<extent unit="page">
<start>11887</start>
<end>11912</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Model Criticism for Long-Form Text Generation
%A Deng, Yuntian
%A Kuleshov, Volodymyr
%A Rush, Alexander
%Y Goldberg, Yoav
%Y Kozareva, Zornitsa
%Y Zhang, Yue
%S Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing
%D 2022
%8 December
%I Association for Computational Linguistics
%C Abu Dhabi, United Arab Emirates
%F deng-etal-2022-model
%X Language models have demonstrated the ability to generate highly fluent text; however, it remains unclear whether their output retains coherent high-level structure (e.g., story progression). Here, we propose to apply a statistical tool, model criticism in latent space, to evaluate the high-level structure of the generated text. Model criticism compares the distributions between real and generated data in a latent space obtained according to an assumptive generative process. Different generative processes identify specific failure modes of the underlying model. We perform experiments on three representative aspects of high-level discourse—coherence, coreference, and topicality—and find that transformer-based language models are able to capture topical structures but have a harder time maintaining structural coherence or modeling coreference.
%R 10.18653/v1/2022.emnlp-main.815
%U https://aclanthology.org/2022.emnlp-main.815
%U https://doi.org/10.18653/v1/2022.emnlp-main.815
%P 11887-11912
Markdown (Informal)
[Model Criticism for Long-Form Text Generation](https://aclanthology.org/2022.emnlp-main.815) (Deng et al., EMNLP 2022)
ACL
- Yuntian Deng, Volodymyr Kuleshov, and Alexander Rush. 2022. Model Criticism for Long-Form Text Generation. In Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, pages 11887–11912, Abu Dhabi, United Arab Emirates. Association for Computational Linguistics.