@article{li-fowlie-2025-gpt,
title = "{GPT} makes a poor {AMR} parser",
author = "Li, Yanming and
Fowlie, Meaghan",
editor = "Tu, Ngoc Duyen Tanja and
Brunner, Annelen and
Lang, Christian",
journal = "Journal for Language Technology and Computational Linguistics",
volume = "38",
number = "2",
month = jul,
year = "2025",
publisher = "German Society for Computational Linguistics and Language Technology",
url = "https://aclanthology.org/2025.jlcl-2.4/",
doi = "10.21248/jlcl.38.2025.285",
pages = "43--76",
abstract = "This paper evaluates GPT models as out-of-the-box Abstract Meaning Representation (AMR) parsers using prompt-based strategies, including 0-shot, few-shot, Chain-of-Thought (CoT), and a two-step approach in which core arguments and non-core roles are handled separately. Our results show that GPT-3.5 and GPT-4o fall well short of state-of-the-art parsers, with a maximum Smatch score of 60 using GPT-4o in a 5-shot setting. While CoT prompting provides some interpretability, it does not improve performance. We further conduct fine-grained evaluations, revealing GPT{'}s limited ability to handle AMR-specific linguistic structures and complex semantic roles. Our findings suggest that, despite recent advances, GPT models are not yet suitable as standalone AMR parsers."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-fowlie-2025-gpt">
<titleInfo>
<title>GPT makes a poor AMR parser</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yanming</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Meaghan</namePart>
<namePart type="family">Fowlie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2025-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<genre authority="bibutilsgt">journal article</genre>
<relatedItem type="host">
<titleInfo>
<title>Journal for Language Technology and Computational Linguistics</title>
</titleInfo>
<originInfo>
<issuance>continuing</issuance>
<publisher>German Society for Computational Linguistics and Language Technology</publisher>
</originInfo>
<genre authority="marcgt">periodical</genre>
<genre authority="bibutilsgt">academic journal</genre>
</relatedItem>
<abstract>This paper evaluates GPT models as out-of-the-box Abstract Meaning Representation (AMR) parsers using prompt-based strategies, including 0-shot, few-shot, Chain-of-Thought (CoT), and a two-step approach in which core arguments and non-core roles are handled separately. Our results show that GPT-3.5 and GPT-4o fall well short of state-of-the-art parsers, with a maximum Smatch score of 60 using GPT-4o in a 5-shot setting. While CoT prompting provides some interpretability, it does not improve performance. We further conduct fine-grained evaluations, revealing GPT’s limited ability to handle AMR-specific linguistic structures and complex semantic roles. Our findings suggest that, despite recent advances, GPT models are not yet suitable as standalone AMR parsers.</abstract>
<identifier type="citekey">li-fowlie-2025-gpt</identifier>
<identifier type="doi">10.21248/jlcl.38.2025.285</identifier>
<location>
<url>https://aclanthology.org/2025.jlcl-2.4/</url>
</location>
<part>
<date>2025-07</date>
<detail type="volume"><number>38</number></detail>
<detail type="issue"><number>2</number></detail>
<extent unit="page">
<start>43</start>
<end>76</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Journal Article
%T GPT makes a poor AMR parser
%A Li, Yanming
%A Fowlie, Meaghan
%J Journal for Language Technology and Computational Linguistics
%D 2025
%8 July
%V 38
%N 2
%I German Society for Computational Linguistics and Language Technology
%F li-fowlie-2025-gpt
%X This paper evaluates GPT models as out-of-the-box Abstract Meaning Representation (AMR) parsers using prompt-based strategies, including 0-shot, few-shot, Chain-of-Thought (CoT), and a two-step approach in which core arguments and non-core roles are handled separately. Our results show that GPT-3.5 and GPT-4o fall well short of state-of-the-art parsers, with a maximum Smatch score of 60 using GPT-4o in a 5-shot setting. While CoT prompting provides some interpretability, it does not improve performance. We further conduct fine-grained evaluations, revealing GPT’s limited ability to handle AMR-specific linguistic structures and complex semantic roles. Our findings suggest that, despite recent advances, GPT models are not yet suitable as standalone AMR parsers.
%R 10.21248/jlcl.38.2025.285
%U https://aclanthology.org/2025.jlcl-2.4/
%U https://doi.org/10.21248/jlcl.38.2025.285
%P 43-76
Markdown (Informal)
[GPT makes a poor AMR parser](https://aclanthology.org/2025.jlcl-2.4/) (Li & Fowlie, JLCL 2025)
ACL
- Yanming Li and Meaghan Fowlie. 2025. GPT makes a poor AMR parser. Journal for Language Technology and Computational Linguistics, 38(2):43–76.