@inproceedings{schlegel-etal-2022-transformers,
title = "Can Transformers Reason in Fragments of Natural Language?",
author = "Schlegel, Viktor and
Pavlov, Kamen and
Pratt-Hartmann, Ian",
editor = "Goldberg, Yoav and
Kozareva, Zornitsa and
Zhang, Yue",
booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",
month = dec,
year = "2022",
address = "Abu Dhabi, United Arab Emirates",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2022.emnlp-main.768",
doi = "10.18653/v1/2022.emnlp-main.768",
pages = "11184--11199",
abstract = "State-of-the-art deep-learning-based approaches to Natural Language Processing (NLP) are credited with various capabilities that involve reasoning with natural language texts. {\%}However, reasoning in this setting is often ill-defined and shallow. In this paper we carry out a large-scale empirical study investigating the detection of formally valid inferences in controlled fragments of natural language for which the satisfiability problem becomes increasingly complex. We find that, while transformer-based language models perform surprisingly well in these scenarios, a deeper analysis reveals that they appear to overfit to superficial patterns in the data rather than acquiring the logical principles governing the reasoning in these fragments.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="schlegel-etal-2022-transformers">
<titleInfo>
<title>Can Transformers Reason in Fragments of Natural Language?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Viktor</namePart>
<namePart type="family">Schlegel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kamen</namePart>
<namePart type="family">Pavlov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ian</namePart>
<namePart type="family">Pratt-Hartmann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2022-12</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yoav</namePart>
<namePart type="family">Goldberg</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zornitsa</namePart>
<namePart type="family">Kozareva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Abu Dhabi, United Arab Emirates</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>State-of-the-art deep-learning-based approaches to Natural Language Processing (NLP) are credited with various capabilities that involve reasoning with natural language texts. %However, reasoning in this setting is often ill-defined and shallow. In this paper we carry out a large-scale empirical study investigating the detection of formally valid inferences in controlled fragments of natural language for which the satisfiability problem becomes increasingly complex. We find that, while transformer-based language models perform surprisingly well in these scenarios, a deeper analysis reveals that they appear to overfit to superficial patterns in the data rather than acquiring the logical principles governing the reasoning in these fragments.</abstract>
<identifier type="citekey">schlegel-etal-2022-transformers</identifier>
<identifier type="doi">10.18653/v1/2022.emnlp-main.768</identifier>
<location>
<url>https://aclanthology.org/2022.emnlp-main.768</url>
</location>
<part>
<date>2022-12</date>
<extent unit="page">
<start>11184</start>
<end>11199</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Can Transformers Reason in Fragments of Natural Language?
%A Schlegel, Viktor
%A Pavlov, Kamen
%A Pratt-Hartmann, Ian
%Y Goldberg, Yoav
%Y Kozareva, Zornitsa
%Y Zhang, Yue
%S Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing
%D 2022
%8 December
%I Association for Computational Linguistics
%C Abu Dhabi, United Arab Emirates
%F schlegel-etal-2022-transformers
%X State-of-the-art deep-learning-based approaches to Natural Language Processing (NLP) are credited with various capabilities that involve reasoning with natural language texts. %However, reasoning in this setting is often ill-defined and shallow. In this paper we carry out a large-scale empirical study investigating the detection of formally valid inferences in controlled fragments of natural language for which the satisfiability problem becomes increasingly complex. We find that, while transformer-based language models perform surprisingly well in these scenarios, a deeper analysis reveals that they appear to overfit to superficial patterns in the data rather than acquiring the logical principles governing the reasoning in these fragments.
%R 10.18653/v1/2022.emnlp-main.768
%U https://aclanthology.org/2022.emnlp-main.768
%U https://doi.org/10.18653/v1/2022.emnlp-main.768
%P 11184-11199
Markdown (Informal)
[Can Transformers Reason in Fragments of Natural Language?](https://aclanthology.org/2022.emnlp-main.768) (Schlegel et al., EMNLP 2022)
ACL
- Viktor Schlegel, Kamen Pavlov, and Ian Pratt-Hartmann. 2022. Can Transformers Reason in Fragments of Natural Language?. In Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, pages 11184–11199, Abu Dhabi, United Arab Emirates. Association for Computational Linguistics.