@inproceedings{miro-maestre-martinez-murillo-2026-evaluating,
title = "Evaluating Machine Translation and Automatic Metrics in Subtitling: A Case Study on {S}panish Multiword Expressions",
author = "Mir{\'o} Maestre, Mar{\'i}a and
Mart{\'i}nez-Murillo, Iv{\'a}n",
editor = "Shterionov, Dimitar and
Vanmassenhove, Eva and
De Sisto, Mirella and
Blain, Fred and
Pourmostafa Roshan Sharami, Javad and
Lepp, Lisa and
Manna, Chiara and
Rescigno, Argentina Anna and
Karakanta, Alina and
Rigouts Terryn, Ayla and
Lardelli, Manuel and
Resende, Natalia and
Murgolo, Elena and
Hackenbuchner, Jani{\c{c}}a and
Zaretskaya, Anna and
Espl{\`a}-Gomis, Miquel and
Etchegoyhen, Thierry and
Gromann, Dagmar and
Bawden, Rachel and
Haddow, Barry and
Szoc, Sara and
Forcada, Mikel and
Moniz, Helena",
booktitle = "Proceedings of the 26th Annual Conference of the {E}uropean Association for Machine Translation (Volume 1)",
month = jun,
year = "2026",
address = "Tilburg, The Netherlands",
publisher = "European Association for Machine Translation",
url = "https://aclanthology.org/2026.eamt-1.29/",
pages = "459--472",
ISBN = "9789403901411",
abstract = "Evaluating the translation of multi-word expressions (MWEs) remains a major challenge for Machine Translation (MT), particularly in audiovisual subtitling, where idiomatic meaning and cultural context are essential for adequacy. This study investigates both the ability of state-of-the-art MT systems to translate Spanish MWEs into English and the extent to which current automatic evaluation methods reflect expert human judgment. We introduce \textsc{ALMO-MWE}, a dataset of 235 MWEs extracted from four films by Pedro Almod{\'o}var to evaluate four MT systems using automatic metrics, LLM-as-a-judge approaches, and professional human assessment. Our results reveal a substantial mismatch between traditional automatic metrics and human judgments: n-gram-based metrics show near-zero correlation with expert evaluation and only limited discriminative capacity. In contrast, neural metrics and LLM-based judges exhibit substantially stronger alignment with human assessments, with GPT-OSS achieving the highest overall correlation. These findings highlight fundamental limitations of surface-form metrics for culturally and contextually sensitive translation phenomena and underscore the need for context-aware evaluation frameworks when assessing the translation quality of MWEs in audiovisual translation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="miro-maestre-martinez-murillo-2026-evaluating">
<titleInfo>
<title>Evaluating Machine Translation and Automatic Metrics in Subtitling: A Case Study on Spanish Multiword Expressions</title>
</titleInfo>
<name type="personal">
<namePart type="given">María</namePart>
<namePart type="family">Miró Maestre</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Iván</namePart>
<namePart type="family">Martínez-Murillo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-06</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 26th Annual Conference of the European Association for Machine Translation (Volume 1)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dimitar</namePart>
<namePart type="family">Shterionov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Eva</namePart>
<namePart type="family">Vanmassenhove</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mirella</namePart>
<namePart type="family">De Sisto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fred</namePart>
<namePart type="family">Blain</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Javad</namePart>
<namePart type="family">Pourmostafa Roshan Sharami</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lisa</namePart>
<namePart type="family">Lepp</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chiara</namePart>
<namePart type="family">Manna</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Argentina</namePart>
<namePart type="given">Anna</namePart>
<namePart type="family">Rescigno</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alina</namePart>
<namePart type="family">Karakanta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayla</namePart>
<namePart type="family">Rigouts Terryn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Manuel</namePart>
<namePart type="family">Lardelli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Natalia</namePart>
<namePart type="family">Resende</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elena</namePart>
<namePart type="family">Murgolo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Janiça</namePart>
<namePart type="family">Hackenbuchner</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="family">Zaretskaya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Miquel</namePart>
<namePart type="family">Esplà-Gomis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thierry</namePart>
<namePart type="family">Etchegoyhen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dagmar</namePart>
<namePart type="family">Gromann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rachel</namePart>
<namePart type="family">Bawden</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barry</namePart>
<namePart type="family">Haddow</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sara</namePart>
<namePart type="family">Szoc</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mikel</namePart>
<namePart type="family">Forcada</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Helena</namePart>
<namePart type="family">Moniz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>European Association for Machine Translation</publisher>
<place>
<placeTerm type="text">Tilburg, The Netherlands</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">9789403901411</identifier>
</relatedItem>
<abstract>Evaluating the translation of multi-word expressions (MWEs) remains a major challenge for Machine Translation (MT), particularly in audiovisual subtitling, where idiomatic meaning and cultural context are essential for adequacy. This study investigates both the ability of state-of-the-art MT systems to translate Spanish MWEs into English and the extent to which current automatic evaluation methods reflect expert human judgment. We introduce ALMO-MWE, a dataset of 235 MWEs extracted from four films by Pedro Almodóvar to evaluate four MT systems using automatic metrics, LLM-as-a-judge approaches, and professional human assessment. Our results reveal a substantial mismatch between traditional automatic metrics and human judgments: n-gram-based metrics show near-zero correlation with expert evaluation and only limited discriminative capacity. In contrast, neural metrics and LLM-based judges exhibit substantially stronger alignment with human assessments, with GPT-OSS achieving the highest overall correlation. These findings highlight fundamental limitations of surface-form metrics for culturally and contextually sensitive translation phenomena and underscore the need for context-aware evaluation frameworks when assessing the translation quality of MWEs in audiovisual translation.</abstract>
<identifier type="citekey">miro-maestre-martinez-murillo-2026-evaluating</identifier>
<location>
<url>https://aclanthology.org/2026.eamt-1.29/</url>
</location>
<part>
<date>2026-06</date>
<extent unit="page">
<start>459</start>
<end>472</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating Machine Translation and Automatic Metrics in Subtitling: A Case Study on Spanish Multiword Expressions
%A Miró Maestre, María
%A Martínez-Murillo, Iván
%Y Shterionov, Dimitar
%Y Vanmassenhove, Eva
%Y De Sisto, Mirella
%Y Blain, Fred
%Y Pourmostafa Roshan Sharami, Javad
%Y Lepp, Lisa
%Y Manna, Chiara
%Y Rescigno, Argentina Anna
%Y Karakanta, Alina
%Y Rigouts Terryn, Ayla
%Y Lardelli, Manuel
%Y Resende, Natalia
%Y Murgolo, Elena
%Y Hackenbuchner, Janiça
%Y Zaretskaya, Anna
%Y Esplà-Gomis, Miquel
%Y Etchegoyhen, Thierry
%Y Gromann, Dagmar
%Y Bawden, Rachel
%Y Haddow, Barry
%Y Szoc, Sara
%Y Forcada, Mikel
%Y Moniz, Helena
%S Proceedings of the 26th Annual Conference of the European Association for Machine Translation (Volume 1)
%D 2026
%8 June
%I European Association for Machine Translation
%C Tilburg, The Netherlands
%@ 9789403901411
%F miro-maestre-martinez-murillo-2026-evaluating
%X Evaluating the translation of multi-word expressions (MWEs) remains a major challenge for Machine Translation (MT), particularly in audiovisual subtitling, where idiomatic meaning and cultural context are essential for adequacy. This study investigates both the ability of state-of-the-art MT systems to translate Spanish MWEs into English and the extent to which current automatic evaluation methods reflect expert human judgment. We introduce ALMO-MWE, a dataset of 235 MWEs extracted from four films by Pedro Almodóvar to evaluate four MT systems using automatic metrics, LLM-as-a-judge approaches, and professional human assessment. Our results reveal a substantial mismatch between traditional automatic metrics and human judgments: n-gram-based metrics show near-zero correlation with expert evaluation and only limited discriminative capacity. In contrast, neural metrics and LLM-based judges exhibit substantially stronger alignment with human assessments, with GPT-OSS achieving the highest overall correlation. These findings highlight fundamental limitations of surface-form metrics for culturally and contextually sensitive translation phenomena and underscore the need for context-aware evaluation frameworks when assessing the translation quality of MWEs in audiovisual translation.
%U https://aclanthology.org/2026.eamt-1.29/
%P 459-472
Markdown (Informal)
[Evaluating Machine Translation and Automatic Metrics in Subtitling: A Case Study on Spanish Multiword Expressions](https://aclanthology.org/2026.eamt-1.29/) (Miró Maestre & Martínez-Murillo, EAMT 2026)
ACL