@inproceedings{thavarasa-etal-2026-obligatory,
title = "Obligatory Slots: Under Reference-Free Evaluation, Dropping a Distinction the Source Never Made Is Free",
author = "Thavarasa, Luxshan and
Thevakumar, Jubeerathan and
Sukumar, Sivasuthan",
editor = "Haddow, Barry and
Kocmi, Tom and
Koehn, Philipp and
Monz, Christof",
booktitle = "Proceedings of the Eleventh Conference on Machine Translation",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.wmt-main.41/",
pages = "727--746",
ISBN = "979-8-89176-474-3",
abstract = "When the target language must mark a distinction the source does not, a translation system must still pick a value. English they came becomes one Tamil verb if those who came are people and another if they are not; English does not say which. We ask what automatic evaluation charges when the choice is wrong, or never made. We release TamilLingBench, an English{\textrightarrow}Tamil challenge set for three verb-agreement distinctions, checked by a morphological analyser: rationality (திணை tiṇai), gender, and number as a control. We pre-registered a prediction that neural metrics would not notice a single wrong morpheme. It is wrong for the neural reference-based metrics: COMET-22 charges 8.52{\texttimes} the score difference that human raters reliably notice. On the systems' own errors, both reference-free metrics rank the wrong form first more often than not, and a legitimate Tamil form that leaves the distinction out is not charged at all. Under reference-free evaluation, dropping the distinction is free; reference-based metrics do charge for it. xCOMET finds the error{'}s region but not the morpheme. Inside one model, activation patching finds the same asymmetry: a distinction English marks is settled by the middle of the network, one it leaves unmarked only near the output."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="thavarasa-etal-2026-obligatory">
<titleInfo>
<title>Obligatory Slots: Under Reference-Free Evaluation, Dropping a Distinction the Source Never Made Is Free</title>
</titleInfo>
<name type="personal">
<namePart type="given">Luxshan</namePart>
<namePart type="family">Thavarasa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jubeerathan</namePart>
<namePart type="family">Thevakumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sivasuthan</namePart>
<namePart type="family">Sukumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Eleventh Conference on Machine Translation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Barry</namePart>
<namePart type="family">Haddow</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tom</namePart>
<namePart type="family">Kocmi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Philipp</namePart>
<namePart type="family">Koehn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christof</namePart>
<namePart type="family">Monz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-474-3</identifier>
</relatedItem>
<abstract>When the target language must mark a distinction the source does not, a translation system must still pick a value. English they came becomes one Tamil verb if those who came are people and another if they are not; English does not say which. We ask what automatic evaluation charges when the choice is wrong, or never made. We release TamilLingBench, an English→Tamil challenge set for three verb-agreement distinctions, checked by a morphological analyser: rationality (திணை tiṇai), gender, and number as a control. We pre-registered a prediction that neural metrics would not notice a single wrong morpheme. It is wrong for the neural reference-based metrics: COMET-22 charges 8.52× the score difference that human raters reliably notice. On the systems’ own errors, both reference-free metrics rank the wrong form first more often than not, and a legitimate Tamil form that leaves the distinction out is not charged at all. Under reference-free evaluation, dropping the distinction is free; reference-based metrics do charge for it. xCOMET finds the error’s region but not the morpheme. Inside one model, activation patching finds the same asymmetry: a distinction English marks is settled by the middle of the network, one it leaves unmarked only near the output.</abstract>
<identifier type="citekey">thavarasa-etal-2026-obligatory</identifier>
<location>
<url>https://aclanthology.org/2026.wmt-main.41/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>727</start>
<end>746</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Obligatory Slots: Under Reference-Free Evaluation, Dropping a Distinction the Source Never Made Is Free
%A Thavarasa, Luxshan
%A Thevakumar, Jubeerathan
%A Sukumar, Sivasuthan
%Y Haddow, Barry
%Y Kocmi, Tom
%Y Koehn, Philipp
%Y Monz, Christof
%S Proceedings of the Eleventh Conference on Machine Translation
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-474-3
%F thavarasa-etal-2026-obligatory
%X When the target language must mark a distinction the source does not, a translation system must still pick a value. English they came becomes one Tamil verb if those who came are people and another if they are not; English does not say which. We ask what automatic evaluation charges when the choice is wrong, or never made. We release TamilLingBench, an English→Tamil challenge set for three verb-agreement distinctions, checked by a morphological analyser: rationality (திணை tiṇai), gender, and number as a control. We pre-registered a prediction that neural metrics would not notice a single wrong morpheme. It is wrong for the neural reference-based metrics: COMET-22 charges 8.52× the score difference that human raters reliably notice. On the systems’ own errors, both reference-free metrics rank the wrong form first more often than not, and a legitimate Tamil form that leaves the distinction out is not charged at all. Under reference-free evaluation, dropping the distinction is free; reference-based metrics do charge for it. xCOMET finds the error’s region but not the morpheme. Inside one model, activation patching finds the same asymmetry: a distinction English marks is settled by the middle of the network, one it leaves unmarked only near the output.
%U https://aclanthology.org/2026.wmt-main.41/
%P 727-746
Markdown (Informal)
[Obligatory Slots: Under Reference-Free Evaluation, Dropping a Distinction the Source Never Made Is Free](https://aclanthology.org/2026.wmt-main.41/) (Thavarasa et al., WMT 2026)
ACL