@inproceedings{eckhardt-etal-2026-phrase,
title = "Phrase at {WMT}26 Automated {MT} Evaluation Task",
author = "Eckhardt, Alan and
Glembek, Ondrej and
Stewart, Craig",
editor = "Haddow, Barry and
Kocmi, Tom and
Koehn, Philipp and
Monz, Christof",
booktitle = "Proceedings of the Eleventh Conference on Machine Translation",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.wmt-main.94/",
pages = "1615--1623",
ISBN = "979-8-89176-474-3",
abstract = "We describe the Phrase submission to WMT26{'}s shared task on automated MT quality evaluation, covering all three of its tasks: segment-level error detection and span annotation (T1), segment-level quality score prediction (T2), and detection of error-free segments (T3). Treating independent LLMs as a panel of ESA (Error Span Annotation)-style error-span judges, we measure a ceiling: a perfect filter over the panel{'}s pooled spans would raise MPP-F (our span-matching F-score) by roughly two thirds over the strongest single judge. An exhaustive search for how to realize that headroom (diversity, rank-weighted aggregation, a per-judge reliability prior, and boundary refinement) never beats the strongest single judge on MPP-F (T1), because the judges' shared recall is too low for any gold-free span selection to close the gap. The two tasks derived from the same spans behave differently: a static reliability prior wins the segment score (T2), and simple agreement wins the binary error-free decision (T3). Our primary submission, Phrase01, follows this split; a secondary submission, Phrase02, forces in a third judge (UFAL{'}s cat-v4) across all tasks as an explicit ablation, despite evidence it does not help. The lever for span-level detection is annotation coverage through generation, not aggregation; for the derived tasks, it is matching the aggregation mechanism to the metric."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="eckhardt-etal-2026-phrase">
<titleInfo>
<title>Phrase at WMT26 Automated MT Evaluation Task</title>
</titleInfo>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Eckhardt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ondrej</namePart>
<namePart type="family">Glembek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Craig</namePart>
<namePart type="family">Stewart</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Eleventh Conference on Machine Translation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Barry</namePart>
<namePart type="family">Haddow</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tom</namePart>
<namePart type="family">Kocmi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Philipp</namePart>
<namePart type="family">Koehn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christof</namePart>
<namePart type="family">Monz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-474-3</identifier>
</relatedItem>
<abstract>We describe the Phrase submission to WMT26’s shared task on automated MT quality evaluation, covering all three of its tasks: segment-level error detection and span annotation (T1), segment-level quality score prediction (T2), and detection of error-free segments (T3). Treating independent LLMs as a panel of ESA (Error Span Annotation)-style error-span judges, we measure a ceiling: a perfect filter over the panel’s pooled spans would raise MPP-F (our span-matching F-score) by roughly two thirds over the strongest single judge. An exhaustive search for how to realize that headroom (diversity, rank-weighted aggregation, a per-judge reliability prior, and boundary refinement) never beats the strongest single judge on MPP-F (T1), because the judges’ shared recall is too low for any gold-free span selection to close the gap. The two tasks derived from the same spans behave differently: a static reliability prior wins the segment score (T2), and simple agreement wins the binary error-free decision (T3). Our primary submission, Phrase01, follows this split; a secondary submission, Phrase02, forces in a third judge (UFAL’s cat-v4) across all tasks as an explicit ablation, despite evidence it does not help. The lever for span-level detection is annotation coverage through generation, not aggregation; for the derived tasks, it is matching the aggregation mechanism to the metric.</abstract>
<identifier type="citekey">eckhardt-etal-2026-phrase</identifier>
<location>
<url>https://aclanthology.org/2026.wmt-main.94/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1615</start>
<end>1623</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Phrase at WMT26 Automated MT Evaluation Task
%A Eckhardt, Alan
%A Glembek, Ondrej
%A Stewart, Craig
%Y Haddow, Barry
%Y Kocmi, Tom
%Y Koehn, Philipp
%Y Monz, Christof
%S Proceedings of the Eleventh Conference on Machine Translation
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-474-3
%F eckhardt-etal-2026-phrase
%X We describe the Phrase submission to WMT26’s shared task on automated MT quality evaluation, covering all three of its tasks: segment-level error detection and span annotation (T1), segment-level quality score prediction (T2), and detection of error-free segments (T3). Treating independent LLMs as a panel of ESA (Error Span Annotation)-style error-span judges, we measure a ceiling: a perfect filter over the panel’s pooled spans would raise MPP-F (our span-matching F-score) by roughly two thirds over the strongest single judge. An exhaustive search for how to realize that headroom (diversity, rank-weighted aggregation, a per-judge reliability prior, and boundary refinement) never beats the strongest single judge on MPP-F (T1), because the judges’ shared recall is too low for any gold-free span selection to close the gap. The two tasks derived from the same spans behave differently: a static reliability prior wins the segment score (T2), and simple agreement wins the binary error-free decision (T3). Our primary submission, Phrase01, follows this split; a secondary submission, Phrase02, forces in a third judge (UFAL’s cat-v4) across all tasks as an explicit ablation, despite evidence it does not help. The lever for span-level detection is annotation coverage through generation, not aggregation; for the derived tasks, it is matching the aggregation mechanism to the metric.
%U https://aclanthology.org/2026.wmt-main.94/
%P 1615-1623
Markdown (Informal)
[Phrase at WMT26 Automated MT Evaluation Task](https://aclanthology.org/2026.wmt-main.94/) (Eckhardt et al., WMT 2026)
ACL
- Alan Eckhardt, Ondrej Glembek, and Craig Stewart. 2026. Phrase at WMT26 Automated MT Evaluation Task. In Proceedings of the Eleventh Conference on Machine Translation, pages 1615–1623, Budapest, Hungary. Association for Computational Linguistics.