@inproceedings{yanishevsky-norris-2026-llm,
title = "{LLM}-as-a-Jury for Machine Translation Publishability Assessment",
author = "Yanishevsky, Alex and
Norris, Olivia",
editor = "Shterionov, Dimitar and
Vanmassenhove, Eva and
De Sisto, Mirella and
Blain, Fred and
Pourmostafa Roshan Sharami, Javad and
Lepp, Lisa and
Manna, Chiara and
Rescigno, Argentina Anna and
Karakanta, Alina and
Rigouts Terryn, Ayla and
Lardelli, Manuel and
Resende, Natalia and
Murgolo, Elena and
Hackenbuchner, Jani{\c{c}}a and
Zaretskaya, Anna and
Espl{\`a}-Gomis, Miquel and
Etchegoyhen, Thierry and
Gromann, Dagmar and
Bawden, Rachel and
Haddow, Barry and
Szoc, Sara and
Forcada, Mikel and
Moniz, Helena",
booktitle = "Proceedings of the 26th Annual Conference of the {E}uropean Association for Machine Translation (Volume 1)",
month = jun,
year = "2026",
address = "Tilburg, The Netherlands",
publisher = "European Association for Machine Translation",
url = "https://aclanthology.org/2026.eamt-1.7/",
pages = "76--84",
ISBN = "9789403901411",
abstract = "We propose an LLM-as-a-Jury framework for determining machine translation publishability, aggregating judgments from multiple large language models via logistic regression rather than relying on a single judge. Publishability is defined as the absence of major or critical errors{---}those that render a translation unsuitable for public release without human post-editing. We compare three evaluation frameworks: a generic Edit Effort Estimation (EEE) prompt based on lexical accuracy, grammatical correctness and semantic coherence, a generic Linguistic Quality Assurance (LQA) prompt based on the MQM error taxonomy, and a purpose-built Publishability prompt optimized via DSPy and augmented with domain-specific fine-tuning. Experiments across three domains and nine language pairs show that (i) the jury ensemble matches or outperforms the best individual juror in nearly every condition, (ii) EEE and LQA juries are competitive with and occasionally exceed the Publishability jury on macro-F1, (iii) the Publishability framework offers stronger precision and a more favorable error correction asymmetry, and (iv) domain-specific fine-tuning yields substantial recall gains in client-heavy domains. These results support the viability of fully automated publishability determination in enterprise MT workflows."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yanishevsky-norris-2026-llm">
<titleInfo>
<title>LLM-as-a-Jury for Machine Translation Publishability Assessment</title>
</titleInfo>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Yanishevsky</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Olivia</namePart>
<namePart type="family">Norris</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-06</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 26th Annual Conference of the European Association for Machine Translation (Volume 1)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dimitar</namePart>
<namePart type="family">Shterionov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Eva</namePart>
<namePart type="family">Vanmassenhove</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mirella</namePart>
<namePart type="family">De Sisto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fred</namePart>
<namePart type="family">Blain</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Javad</namePart>
<namePart type="family">Pourmostafa Roshan Sharami</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lisa</namePart>
<namePart type="family">Lepp</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chiara</namePart>
<namePart type="family">Manna</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Argentina</namePart>
<namePart type="given">Anna</namePart>
<namePart type="family">Rescigno</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alina</namePart>
<namePart type="family">Karakanta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayla</namePart>
<namePart type="family">Rigouts Terryn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Manuel</namePart>
<namePart type="family">Lardelli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Natalia</namePart>
<namePart type="family">Resende</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elena</namePart>
<namePart type="family">Murgolo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Janiça</namePart>
<namePart type="family">Hackenbuchner</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="family">Zaretskaya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Miquel</namePart>
<namePart type="family">Esplà-Gomis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thierry</namePart>
<namePart type="family">Etchegoyhen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dagmar</namePart>
<namePart type="family">Gromann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rachel</namePart>
<namePart type="family">Bawden</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barry</namePart>
<namePart type="family">Haddow</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sara</namePart>
<namePart type="family">Szoc</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mikel</namePart>
<namePart type="family">Forcada</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Helena</namePart>
<namePart type="family">Moniz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>European Association for Machine Translation</publisher>
<place>
<placeTerm type="text">Tilburg, The Netherlands</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">9789403901411</identifier>
</relatedItem>
<abstract>We propose an LLM-as-a-Jury framework for determining machine translation publishability, aggregating judgments from multiple large language models via logistic regression rather than relying on a single judge. Publishability is defined as the absence of major or critical errors—those that render a translation unsuitable for public release without human post-editing. We compare three evaluation frameworks: a generic Edit Effort Estimation (EEE) prompt based on lexical accuracy, grammatical correctness and semantic coherence, a generic Linguistic Quality Assurance (LQA) prompt based on the MQM error taxonomy, and a purpose-built Publishability prompt optimized via DSPy and augmented with domain-specific fine-tuning. Experiments across three domains and nine language pairs show that (i) the jury ensemble matches or outperforms the best individual juror in nearly every condition, (ii) EEE and LQA juries are competitive with and occasionally exceed the Publishability jury on macro-F1, (iii) the Publishability framework offers stronger precision and a more favorable error correction asymmetry, and (iv) domain-specific fine-tuning yields substantial recall gains in client-heavy domains. These results support the viability of fully automated publishability determination in enterprise MT workflows.</abstract>
<identifier type="citekey">yanishevsky-norris-2026-llm</identifier>
<location>
<url>https://aclanthology.org/2026.eamt-1.7/</url>
</location>
<part>
<date>2026-06</date>
<extent unit="page">
<start>76</start>
<end>84</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T LLM-as-a-Jury for Machine Translation Publishability Assessment
%A Yanishevsky, Alex
%A Norris, Olivia
%Y Shterionov, Dimitar
%Y Vanmassenhove, Eva
%Y De Sisto, Mirella
%Y Blain, Fred
%Y Pourmostafa Roshan Sharami, Javad
%Y Lepp, Lisa
%Y Manna, Chiara
%Y Rescigno, Argentina Anna
%Y Karakanta, Alina
%Y Rigouts Terryn, Ayla
%Y Lardelli, Manuel
%Y Resende, Natalia
%Y Murgolo, Elena
%Y Hackenbuchner, Janiça
%Y Zaretskaya, Anna
%Y Esplà-Gomis, Miquel
%Y Etchegoyhen, Thierry
%Y Gromann, Dagmar
%Y Bawden, Rachel
%Y Haddow, Barry
%Y Szoc, Sara
%Y Forcada, Mikel
%Y Moniz, Helena
%S Proceedings of the 26th Annual Conference of the European Association for Machine Translation (Volume 1)
%D 2026
%8 June
%I European Association for Machine Translation
%C Tilburg, The Netherlands
%@ 9789403901411
%F yanishevsky-norris-2026-llm
%X We propose an LLM-as-a-Jury framework for determining machine translation publishability, aggregating judgments from multiple large language models via logistic regression rather than relying on a single judge. Publishability is defined as the absence of major or critical errors—those that render a translation unsuitable for public release without human post-editing. We compare three evaluation frameworks: a generic Edit Effort Estimation (EEE) prompt based on lexical accuracy, grammatical correctness and semantic coherence, a generic Linguistic Quality Assurance (LQA) prompt based on the MQM error taxonomy, and a purpose-built Publishability prompt optimized via DSPy and augmented with domain-specific fine-tuning. Experiments across three domains and nine language pairs show that (i) the jury ensemble matches or outperforms the best individual juror in nearly every condition, (ii) EEE and LQA juries are competitive with and occasionally exceed the Publishability jury on macro-F1, (iii) the Publishability framework offers stronger precision and a more favorable error correction asymmetry, and (iv) domain-specific fine-tuning yields substantial recall gains in client-heavy domains. These results support the viability of fully automated publishability determination in enterprise MT workflows.
%U https://aclanthology.org/2026.eamt-1.7/
%P 76-84
Markdown (Informal)
[LLM-as-a-Jury for Machine Translation Publishability Assessment](https://aclanthology.org/2026.eamt-1.7/) (Yanishevsky & Norris, EAMT 2026)
ACL