@inproceedings{mouratidis-etal-2021-nodeele,
title = "{N}o{D}ee{L}e: A Novel Deep Learning Schema for Evaluating Neural Machine Translation Systems",
author = "Mouratidis, Despoina and
Stasimioti, Maria and
Sosoni, Vilelmini and
Kermanidis, Katia Lida",
editor = "Mitkov, Ruslan and
Sosoni, Vilelmini and
Gigu{\`e}re, Julie Christine and
Murgolo, Elena and
Deysel, Elizabeth",
booktitle = "Proceedings of the Translation and Interpreting Technology Online Conference",
month = jul,
year = "2021",
address = "Held Online",
publisher = "INCOMA Ltd.",
url = "https://aclanthology.org/2021.triton-1.5",
pages = "37--47",
abstract = "Due to the wide-spread development of Machine Translation (MT) systems {--}especially Neural Machine Translation (NMT) systems{--} MT evaluation, both automatic and human, has become more and more important as it helps us establish how MT systems perform. Yet, automatic evaluation metrics have lagged behind, as the most popular choices (e.g., BLEU, METEOR and ROUGE) may correlate poorly with human judgments. This paper seeks to put to the test an evaluation model based on a novel deep learning schema (NoDeeLe) used to compare two NMT systems on four different text genres, i.e. medical, legal, marketing and literary in the English-Greek language pair. The model utilizes information from the source segments, the MT outputs and the reference translation, as well as the automatic metrics BLEU, METEOR and WER. The proposed schema achieves a strong correlation with human judgment (78{\%} average accuracy for the four texts with the highest accuracy, i.e. 85{\%}, observed in the case of the marketing text), while it outperforms classic machine learning algorithms and automatic metrics.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mouratidis-etal-2021-nodeele">
<titleInfo>
<title>NoDeeLe: A Novel Deep Learning Schema for Evaluating Neural Machine Translation Systems</title>
</titleInfo>
<name type="personal">
<namePart type="given">Despoina</namePart>
<namePart type="family">Mouratidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maria</namePart>
<namePart type="family">Stasimioti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vilelmini</namePart>
<namePart type="family">Sosoni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Katia</namePart>
<namePart type="given">Lida</namePart>
<namePart type="family">Kermanidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2021-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Translation and Interpreting Technology Online Conference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ruslan</namePart>
<namePart type="family">Mitkov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vilelmini</namePart>
<namePart type="family">Sosoni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Julie</namePart>
<namePart type="given">Christine</namePart>
<namePart type="family">Giguère</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elena</namePart>
<namePart type="family">Murgolo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elizabeth</namePart>
<namePart type="family">Deysel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>INCOMA Ltd.</publisher>
<place>
<placeTerm type="text">Held Online</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Due to the wide-spread development of Machine Translation (MT) systems –especially Neural Machine Translation (NMT) systems– MT evaluation, both automatic and human, has become more and more important as it helps us establish how MT systems perform. Yet, automatic evaluation metrics have lagged behind, as the most popular choices (e.g., BLEU, METEOR and ROUGE) may correlate poorly with human judgments. This paper seeks to put to the test an evaluation model based on a novel deep learning schema (NoDeeLe) used to compare two NMT systems on four different text genres, i.e. medical, legal, marketing and literary in the English-Greek language pair. The model utilizes information from the source segments, the MT outputs and the reference translation, as well as the automatic metrics BLEU, METEOR and WER. The proposed schema achieves a strong correlation with human judgment (78% average accuracy for the four texts with the highest accuracy, i.e. 85%, observed in the case of the marketing text), while it outperforms classic machine learning algorithms and automatic metrics.</abstract>
<identifier type="citekey">mouratidis-etal-2021-nodeele</identifier>
<location>
<url>https://aclanthology.org/2021.triton-1.5</url>
</location>
<part>
<date>2021-07</date>
<extent unit="page">
<start>37</start>
<end>47</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T NoDeeLe: A Novel Deep Learning Schema for Evaluating Neural Machine Translation Systems
%A Mouratidis, Despoina
%A Stasimioti, Maria
%A Sosoni, Vilelmini
%A Kermanidis, Katia Lida
%Y Mitkov, Ruslan
%Y Sosoni, Vilelmini
%Y Giguère, Julie Christine
%Y Murgolo, Elena
%Y Deysel, Elizabeth
%S Proceedings of the Translation and Interpreting Technology Online Conference
%D 2021
%8 July
%I INCOMA Ltd.
%C Held Online
%F mouratidis-etal-2021-nodeele
%X Due to the wide-spread development of Machine Translation (MT) systems –especially Neural Machine Translation (NMT) systems– MT evaluation, both automatic and human, has become more and more important as it helps us establish how MT systems perform. Yet, automatic evaluation metrics have lagged behind, as the most popular choices (e.g., BLEU, METEOR and ROUGE) may correlate poorly with human judgments. This paper seeks to put to the test an evaluation model based on a novel deep learning schema (NoDeeLe) used to compare two NMT systems on four different text genres, i.e. medical, legal, marketing and literary in the English-Greek language pair. The model utilizes information from the source segments, the MT outputs and the reference translation, as well as the automatic metrics BLEU, METEOR and WER. The proposed schema achieves a strong correlation with human judgment (78% average accuracy for the four texts with the highest accuracy, i.e. 85%, observed in the case of the marketing text), while it outperforms classic machine learning algorithms and automatic metrics.
%U https://aclanthology.org/2021.triton-1.5
%P 37-47
Markdown (Informal)
[NoDeeLe: A Novel Deep Learning Schema for Evaluating Neural Machine Translation Systems](https://aclanthology.org/2021.triton-1.5) (Mouratidis et al., TRITON 2021)
ACL