@inproceedings{huidrom-etal-2026-metric,
title = "Which Metric for Which Script? A Quantified Meta-Evaluation of Automatic Metrics and {LLM} Judges for Low-Resource {I}ndic Machine Translation",
author = "Huidrom, Rudali and
Kumar, Vikas and
Pangsatabam, Hoomexsun and
Das, Pinaki and
Singh, Kshetrimayum Boynao and
Konjengbam, Anand",
editor = "Haddow, Barry and
Kocmi, Tom and
Koehn, Philipp and
Monz, Christof",
booktitle = "Proceedings of the Eleventh Conference on Machine Translation",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.wmt-main.17/",
pages = "281--303",
ISBN = "979-8-89176-474-3",
abstract = "Machine translation shared tasks publish several automatic metrics and rank systems on one of them, assuming the choice is inconsequential. We test this assumption on the WMT 2025 and WMT 2026 Low-Resource Indic Language Translation tasks, where pre-trained multilingual encoders are unavailable for several target scripts and languages. As no human gold standard evaluation exists at scale, we adapt the Quantified Reproducibility Assessment (QRA) framework across 20 WMT26 and 14 WMT25 directions, to measure agreement between six automatic metrics and two LLM judges as independent measuring instruments. We show that QRA{'}s Type I measure fails under this repurposing when instruments sit on different scales, and suggest a rank-based repair. Using a script-controlled natural experiment (Manipuri evaluated in both Bengali and Meitei Mayek) and a null-output validity test, we observe that neural metrics inflate scores for text in scripts outside the encoder{'}s coverage and compress it into a narrow discriminative band. Switching to Meitei Mayek raises BERTScore and COMET by 21.6 and 23.3 points, respectively, though language and content are unchanged, and no metric preserves system ranking; on degenerate output the two retain 0.81 and 0.61 points on English-to-Indic directions. The effect reverses in the opposite direction. Metric choice should follow script coverage, not language identity. Lacking human judgments, our claims concern instrument behaviour, not human preference."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="huidrom-etal-2026-metric">
<titleInfo>
<title>Which Metric for Which Script? A Quantified Meta-Evaluation of Automatic Metrics and LLM Judges for Low-Resource Indic Machine Translation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Rudali</namePart>
<namePart type="family">Huidrom</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vikas</namePart>
<namePart type="family">Kumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hoomexsun</namePart>
<namePart type="family">Pangsatabam</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pinaki</namePart>
<namePart type="family">Das</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kshetrimayum</namePart>
<namePart type="given">Boynao</namePart>
<namePart type="family">Singh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anand</namePart>
<namePart type="family">Konjengbam</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Eleventh Conference on Machine Translation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Barry</namePart>
<namePart type="family">Haddow</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tom</namePart>
<namePart type="family">Kocmi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Philipp</namePart>
<namePart type="family">Koehn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christof</namePart>
<namePart type="family">Monz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-474-3</identifier>
</relatedItem>
<abstract>Machine translation shared tasks publish several automatic metrics and rank systems on one of them, assuming the choice is inconsequential. We test this assumption on the WMT 2025 and WMT 2026 Low-Resource Indic Language Translation tasks, where pre-trained multilingual encoders are unavailable for several target scripts and languages. As no human gold standard evaluation exists at scale, we adapt the Quantified Reproducibility Assessment (QRA) framework across 20 WMT26 and 14 WMT25 directions, to measure agreement between six automatic metrics and two LLM judges as independent measuring instruments. We show that QRA’s Type I measure fails under this repurposing when instruments sit on different scales, and suggest a rank-based repair. Using a script-controlled natural experiment (Manipuri evaluated in both Bengali and Meitei Mayek) and a null-output validity test, we observe that neural metrics inflate scores for text in scripts outside the encoder’s coverage and compress it into a narrow discriminative band. Switching to Meitei Mayek raises BERTScore and COMET by 21.6 and 23.3 points, respectively, though language and content are unchanged, and no metric preserves system ranking; on degenerate output the two retain 0.81 and 0.61 points on English-to-Indic directions. The effect reverses in the opposite direction. Metric choice should follow script coverage, not language identity. Lacking human judgments, our claims concern instrument behaviour, not human preference.</abstract>
<identifier type="citekey">huidrom-etal-2026-metric</identifier>
<location>
<url>https://aclanthology.org/2026.wmt-main.17/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>281</start>
<end>303</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Which Metric for Which Script? A Quantified Meta-Evaluation of Automatic Metrics and LLM Judges for Low-Resource Indic Machine Translation
%A Huidrom, Rudali
%A Kumar, Vikas
%A Pangsatabam, Hoomexsun
%A Das, Pinaki
%A Singh, Kshetrimayum Boynao
%A Konjengbam, Anand
%Y Haddow, Barry
%Y Kocmi, Tom
%Y Koehn, Philipp
%Y Monz, Christof
%S Proceedings of the Eleventh Conference on Machine Translation
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-474-3
%F huidrom-etal-2026-metric
%X Machine translation shared tasks publish several automatic metrics and rank systems on one of them, assuming the choice is inconsequential. We test this assumption on the WMT 2025 and WMT 2026 Low-Resource Indic Language Translation tasks, where pre-trained multilingual encoders are unavailable for several target scripts and languages. As no human gold standard evaluation exists at scale, we adapt the Quantified Reproducibility Assessment (QRA) framework across 20 WMT26 and 14 WMT25 directions, to measure agreement between six automatic metrics and two LLM judges as independent measuring instruments. We show that QRA’s Type I measure fails under this repurposing when instruments sit on different scales, and suggest a rank-based repair. Using a script-controlled natural experiment (Manipuri evaluated in both Bengali and Meitei Mayek) and a null-output validity test, we observe that neural metrics inflate scores for text in scripts outside the encoder’s coverage and compress it into a narrow discriminative band. Switching to Meitei Mayek raises BERTScore and COMET by 21.6 and 23.3 points, respectively, though language and content are unchanged, and no metric preserves system ranking; on degenerate output the two retain 0.81 and 0.61 points on English-to-Indic directions. The effect reverses in the opposite direction. Metric choice should follow script coverage, not language identity. Lacking human judgments, our claims concern instrument behaviour, not human preference.
%U https://aclanthology.org/2026.wmt-main.17/
%P 281-303
Markdown (Informal)
[Which Metric for Which Script? A Quantified Meta-Evaluation of Automatic Metrics and LLM Judges for Low-Resource Indic Machine Translation](https://aclanthology.org/2026.wmt-main.17/) (Huidrom et al., WMT 2026)
ACL