@inproceedings{mecharnia-daquin-2025-performance,
title = "Performance and Limitations of Fine-Tuned {LLM}s in {SPARQL} Query Generation",
author = "Mecharnia, Thamer and
d{'}Aquin, Mathieu",
editor = "Gesese, Genet Asefa and
Sack, Harald and
Paulheim, Heiko and
Merono-Penuela, Albert and
Chen, Lihu",
booktitle = "Proceedings of the Workshop on Generative AI and Knowledge Graphs (GenAIK)",
month = jan,
year = "2025",
address = "Abu Dhabi, UAE",
publisher = "International Committee on Computational Linguistics",
url = "https://aclanthology.org/2025.genaik-1.8/",
pages = "69--77",
abstract = "Generative AI has simplified information access by enabling natural language-driven interactions between users and automated systems. In particular, Question Answering (QA) has emerged as a key application of AI, facilitating efficient access to complex information through dialogue systems and virtual assistants. The Large Language Models (LLMs) combined with Knowledge Graphs (KGs) have further enhanced QA systems, allowing them to not only correctly interpret natural language but also retrieve precise answers from structured data sources such as Wikidata and DBpedia. However, enabling LLMs to generate machine-readable SPARQL queries from natural language questions (NLQs) remains challenging, particularly for complex questions. In this study, we present experiments in fine-tuning LLMs for the task of NLQ-to-SPARQL transformation. We rely on benchmark datasets for training and testing the fine-tuned models, generating queries directly from questions written in English (without further processing of the input or output). By conducting an analytical study, we examine the effectiveness of each model, as well as the limitations associated with using fine-tuned LLMs to generate SPARQL."
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mecharnia-daquin-2025-performance">
<titleInfo>
<title>Performance and Limitations of Fine-Tuned LLMs in SPARQL Query Generation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Thamer</namePart>
<namePart type="family">Mecharnia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mathieu</namePart>
<namePart type="family">d’Aquin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2025-01</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Workshop on Generative AI and Knowledge Graphs (GenAIK)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Genet</namePart>
<namePart type="given">Asefa</namePart>
<namePart type="family">Gesese</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Harald</namePart>
<namePart type="family">Sack</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Heiko</namePart>
<namePart type="family">Paulheim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Albert</namePart>
<namePart type="family">Merono-Penuela</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lihu</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>International Committee on Computational Linguistics</publisher>
<place>
<placeTerm type="text">Abu Dhabi, UAE</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Generative AI has simplified information access by enabling natural language-driven interactions between users and automated systems. In particular, Question Answering (QA) has emerged as a key application of AI, facilitating efficient access to complex information through dialogue systems and virtual assistants. The Large Language Models (LLMs) combined with Knowledge Graphs (KGs) have further enhanced QA systems, allowing them to not only correctly interpret natural language but also retrieve precise answers from structured data sources such as Wikidata and DBpedia. However, enabling LLMs to generate machine-readable SPARQL queries from natural language questions (NLQs) remains challenging, particularly for complex questions. In this study, we present experiments in fine-tuning LLMs for the task of NLQ-to-SPARQL transformation. We rely on benchmark datasets for training and testing the fine-tuned models, generating queries directly from questions written in English (without further processing of the input or output). By conducting an analytical study, we examine the effectiveness of each model, as well as the limitations associated with using fine-tuned LLMs to generate SPARQL.</abstract>
<identifier type="citekey">mecharnia-daquin-2025-performance</identifier>
<location>
<url>https://aclanthology.org/2025.genaik-1.8/</url>
</location>
<part>
<date>2025-01</date>
<extent unit="page">
<start>69</start>
<end>77</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Performance and Limitations of Fine-Tuned LLMs in SPARQL Query Generation
%A Mecharnia, Thamer
%A d’Aquin, Mathieu
%Y Gesese, Genet Asefa
%Y Sack, Harald
%Y Paulheim, Heiko
%Y Merono-Penuela, Albert
%Y Chen, Lihu
%S Proceedings of the Workshop on Generative AI and Knowledge Graphs (GenAIK)
%D 2025
%8 January
%I International Committee on Computational Linguistics
%C Abu Dhabi, UAE
%F mecharnia-daquin-2025-performance
%X Generative AI has simplified information access by enabling natural language-driven interactions between users and automated systems. In particular, Question Answering (QA) has emerged as a key application of AI, facilitating efficient access to complex information through dialogue systems and virtual assistants. The Large Language Models (LLMs) combined with Knowledge Graphs (KGs) have further enhanced QA systems, allowing them to not only correctly interpret natural language but also retrieve precise answers from structured data sources such as Wikidata and DBpedia. However, enabling LLMs to generate machine-readable SPARQL queries from natural language questions (NLQs) remains challenging, particularly for complex questions. In this study, we present experiments in fine-tuning LLMs for the task of NLQ-to-SPARQL transformation. We rely on benchmark datasets for training and testing the fine-tuned models, generating queries directly from questions written in English (without further processing of the input or output). By conducting an analytical study, we examine the effectiveness of each model, as well as the limitations associated with using fine-tuned LLMs to generate SPARQL.
%U https://aclanthology.org/2025.genaik-1.8/
%P 69-77
Markdown (Informal)
[Performance and Limitations of Fine-Tuned LLMs in SPARQL Query Generation](https://aclanthology.org/2025.genaik-1.8/) (Mecharnia & d’Aquin, GenAIK 2025)
ACL