@inproceedings{clifford-etal-2025-gaeilge,
title = "Gaeilge Bhriste {\'o} Shamhlacha Cliste: How Clever Are {LLM}s When Translating {I}rish Text?",
author = "Clifford, Teresa and
Walsh, Abigail and
Davis, Brian and
{\'O} Meachair, M{\'i}che{\'a}l J.",
editor = "Davis, Brian and
Fransen, Theodorus and
Dhonnchadha, Elaine Ui and
Walsh, Abigail",
booktitle = "Proceedings of the 5th Celtic Language Technology Workshop",
month = jan,
year = "2025",
address = "Abu Dhabi [Virtual Workshop]",
publisher = "International Committee on Computational Linguistics",
url = "https://aclanthology.org/2025.cltw-1.5/",
pages = "46--51",
abstract = "Large Language Models have been widely adopted in NLP tasks and applications, how- ever, their ability to accurately process Irish and other minority languages has not been fully explored. In this paper we describe prelim- inary experiments examining the capacity of publicly-available machine translation engines (Google Translate, Microsoft Bing, and eTrans- lation) and prompt-based AI systems systems (ChatGPT 3.5, Llama 2) for translating and handling challenging language features of Irish. A hand-crafted selection of challenging Irish language features were incorporated into trans- lation prompts, and the output from each model was examined by a human evaluator. The re- sults of these experiments indicate that these LLM-based models still struggle with translat- ing rare linguistic phenomena and ambiguous constructions. This preliminary analysis helps to inform further research in this field, pro- viding a simple ranking of publicly-available models, and indicating which language features require particular attention when evaluating model capacity."
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="clifford-etal-2025-gaeilge">
<titleInfo>
<title>Gaeilge Bhriste ó Shamhlacha Cliste: How Clever Are LLMs When Translating Irish Text?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Teresa</namePart>
<namePart type="family">Clifford</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abigail</namePart>
<namePart type="family">Walsh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Brian</namePart>
<namePart type="family">Davis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mícheál</namePart>
<namePart type="given">J</namePart>
<namePart type="family">Ó Meachair</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2025-01</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 5th Celtic Language Technology Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Brian</namePart>
<namePart type="family">Davis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Theodorus</namePart>
<namePart type="family">Fransen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elaine</namePart>
<namePart type="given">Ui</namePart>
<namePart type="family">Dhonnchadha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abigail</namePart>
<namePart type="family">Walsh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>International Committee on Computational Linguistics</publisher>
<place>
<placeTerm type="text">Abu Dhabi [Virtual Workshop]</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Large Language Models have been widely adopted in NLP tasks and applications, how- ever, their ability to accurately process Irish and other minority languages has not been fully explored. In this paper we describe prelim- inary experiments examining the capacity of publicly-available machine translation engines (Google Translate, Microsoft Bing, and eTrans- lation) and prompt-based AI systems systems (ChatGPT 3.5, Llama 2) for translating and handling challenging language features of Irish. A hand-crafted selection of challenging Irish language features were incorporated into trans- lation prompts, and the output from each model was examined by a human evaluator. The re- sults of these experiments indicate that these LLM-based models still struggle with translat- ing rare linguistic phenomena and ambiguous constructions. This preliminary analysis helps to inform further research in this field, pro- viding a simple ranking of publicly-available models, and indicating which language features require particular attention when evaluating model capacity.</abstract>
<identifier type="citekey">clifford-etal-2025-gaeilge</identifier>
<location>
<url>https://aclanthology.org/2025.cltw-1.5/</url>
</location>
<part>
<date>2025-01</date>
<extent unit="page">
<start>46</start>
<end>51</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Gaeilge Bhriste ó Shamhlacha Cliste: How Clever Are LLMs When Translating Irish Text?
%A Clifford, Teresa
%A Walsh, Abigail
%A Davis, Brian
%A Ó Meachair, Mícheál J.
%Y Davis, Brian
%Y Fransen, Theodorus
%Y Dhonnchadha, Elaine Ui
%Y Walsh, Abigail
%S Proceedings of the 5th Celtic Language Technology Workshop
%D 2025
%8 January
%I International Committee on Computational Linguistics
%C Abu Dhabi [Virtual Workshop]
%F clifford-etal-2025-gaeilge
%X Large Language Models have been widely adopted in NLP tasks and applications, how- ever, their ability to accurately process Irish and other minority languages has not been fully explored. In this paper we describe prelim- inary experiments examining the capacity of publicly-available machine translation engines (Google Translate, Microsoft Bing, and eTrans- lation) and prompt-based AI systems systems (ChatGPT 3.5, Llama 2) for translating and handling challenging language features of Irish. A hand-crafted selection of challenging Irish language features were incorporated into trans- lation prompts, and the output from each model was examined by a human evaluator. The re- sults of these experiments indicate that these LLM-based models still struggle with translat- ing rare linguistic phenomena and ambiguous constructions. This preliminary analysis helps to inform further research in this field, pro- viding a simple ranking of publicly-available models, and indicating which language features require particular attention when evaluating model capacity.
%U https://aclanthology.org/2025.cltw-1.5/
%P 46-51
Markdown (Informal)
[Gaeilge Bhriste ó Shamhlacha Cliste: How Clever Are LLMs When Translating Irish Text?](https://aclanthology.org/2025.cltw-1.5/) (Clifford et al., CLTW 2025)
ACL