@article{chang-bergen-2024-language,
title = "Language Model Behavior: A Comprehensive Survey",
author = "Chang, Tyler A. and
Bergen, Benjamin K.",
journal = "Computational Linguistics",
volume = "50",
number = "1",
month = mar,
year = "2024",
address = "Cambridge, MA",
publisher = "MIT Press",
url = "https://aclanthology.org/2024.cl-1.9",
doi = "10.1162/coli_a_00492",
pages = "293--350",
abstract = "Transformer language models have received widespread public attention, yet their generated text is often surprising even to NLP researchers. In this survey, we discuss over 250 recent studies of English language model behavior before task-specific fine-tuning. Language models possess basic capabilities in syntax, semantics, pragmatics, world knowledge, and reasoning, but these capabilities are sensitive to specific inputs and surface features. Despite dramatic increases in generated text quality as models scale to hundreds of billions of parameters, the models are still prone to unfactual responses, commonsense errors, memorized text, and social biases. Many of these weaknesses can be framed as over-generalizations or under-generalizations of learned patterns in text. We synthesize recent results to highlight what is currently known about large language model capabilities, thus providing a resource for applied work and for research in adjacent fields that use language models.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chang-bergen-2024-language">
<titleInfo>
<title>Language Model Behavior: A Comprehensive Survey</title>
</titleInfo>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="given">A</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Benjamin</namePart>
<namePart type="given">K</namePart>
<namePart type="family">Bergen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2024-03</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<genre authority="bibutilsgt">journal article</genre>
<relatedItem type="host">
<titleInfo>
<title>Computational Linguistics</title>
</titleInfo>
<originInfo>
<issuance>continuing</issuance>
<publisher>MIT Press</publisher>
<place>
<placeTerm type="text">Cambridge, MA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">periodical</genre>
<genre authority="bibutilsgt">academic journal</genre>
</relatedItem>
<abstract>Transformer language models have received widespread public attention, yet their generated text is often surprising even to NLP researchers. In this survey, we discuss over 250 recent studies of English language model behavior before task-specific fine-tuning. Language models possess basic capabilities in syntax, semantics, pragmatics, world knowledge, and reasoning, but these capabilities are sensitive to specific inputs and surface features. Despite dramatic increases in generated text quality as models scale to hundreds of billions of parameters, the models are still prone to unfactual responses, commonsense errors, memorized text, and social biases. Many of these weaknesses can be framed as over-generalizations or under-generalizations of learned patterns in text. We synthesize recent results to highlight what is currently known about large language model capabilities, thus providing a resource for applied work and for research in adjacent fields that use language models.</abstract>
<identifier type="citekey">chang-bergen-2024-language</identifier>
<identifier type="doi">10.1162/coli_a_00492</identifier>
<location>
<url>https://aclanthology.org/2024.cl-1.9</url>
</location>
<part>
<date>2024-03</date>
<detail type="volume"><number>50</number></detail>
<detail type="issue"><number>1</number></detail>
<extent unit="page">
<start>293</start>
<end>350</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Journal Article
%T Language Model Behavior: A Comprehensive Survey
%A Chang, Tyler A.
%A Bergen, Benjamin K.
%J Computational Linguistics
%D 2024
%8 March
%V 50
%N 1
%I MIT Press
%C Cambridge, MA
%F chang-bergen-2024-language
%X Transformer language models have received widespread public attention, yet their generated text is often surprising even to NLP researchers. In this survey, we discuss over 250 recent studies of English language model behavior before task-specific fine-tuning. Language models possess basic capabilities in syntax, semantics, pragmatics, world knowledge, and reasoning, but these capabilities are sensitive to specific inputs and surface features. Despite dramatic increases in generated text quality as models scale to hundreds of billions of parameters, the models are still prone to unfactual responses, commonsense errors, memorized text, and social biases. Many of these weaknesses can be framed as over-generalizations or under-generalizations of learned patterns in text. We synthesize recent results to highlight what is currently known about large language model capabilities, thus providing a resource for applied work and for research in adjacent fields that use language models.
%R 10.1162/coli_a_00492
%U https://aclanthology.org/2024.cl-1.9
%U https://doi.org/10.1162/coli_a_00492
%P 293-350
Markdown (Informal)
[Language Model Behavior: A Comprehensive Survey](https://aclanthology.org/2024.cl-1.9) (Chang & Bergen, CL 2024)
ACL