@inproceedings{yadav-etal-2026-finding,
title = "Finding the Gaps in {LLM} Translation with {C}o{ST}",
author = "Yadav, Saumitra and
Mukherjee, Ananya and
Shrivastava, Manish",
editor = "Haddow, Barry and
Kocmi, Tom and
Koehn, Philipp and
Monz, Christof",
booktitle = "Proceedings of the Eleventh Conference on Machine Translation",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.wmt-main.93/",
pages = "1602--1614",
ISBN = "979-8-89176-474-3",
abstract = "Large language models (LLMs) have advanced machine translation substantially, but generic benchmarks often fail to expose where these systems struggle with complex linguistic structures and stylistic nuance. We use CoST (Complex Structures Test), a challenge suite of 1,947 English-Hindi sentences spanning eight genres, autobiography, conversation, legal, mixed, narration, play, poetry, and technical writing, previously introduced for WMT24, to evaluate 22 machine translation systems submitted to the WMT26 General Translation Shared Task for English-Hindi with reference-free, reference-based, and manual evaluation. Our results reveal a consistent genre-level pattern: systems perform well on narrative text such as autobiography and play, but degrade sharply on poetry, legal, and conversational text. We further identify a distinct failure mode in poetry translation, where systems reproduce memorized canonical Hindi verse instead of translating the given English text, and manual analysis surfaces recurring issues in named-entity handling, wrong-language output, and lexical choice, several of which persist from our prior CoST evaluation on WMT24 submissions. Our test suite is available at https://github.com/deciphyre/CoST-WMT-26-Test-Suite-Task."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yadav-etal-2026-finding">
<titleInfo>
<title>Finding the Gaps in LLM Translation with CoST</title>
</titleInfo>
<name type="personal">
<namePart type="given">Saumitra</namePart>
<namePart type="family">Yadav</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ananya</namePart>
<namePart type="family">Mukherjee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Manish</namePart>
<namePart type="family">Shrivastava</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Eleventh Conference on Machine Translation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Barry</namePart>
<namePart type="family">Haddow</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tom</namePart>
<namePart type="family">Kocmi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Philipp</namePart>
<namePart type="family">Koehn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christof</namePart>
<namePart type="family">Monz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-474-3</identifier>
</relatedItem>
<abstract>Large language models (LLMs) have advanced machine translation substantially, but generic benchmarks often fail to expose where these systems struggle with complex linguistic structures and stylistic nuance. We use CoST (Complex Structures Test), a challenge suite of 1,947 English-Hindi sentences spanning eight genres, autobiography, conversation, legal, mixed, narration, play, poetry, and technical writing, previously introduced for WMT24, to evaluate 22 machine translation systems submitted to the WMT26 General Translation Shared Task for English-Hindi with reference-free, reference-based, and manual evaluation. Our results reveal a consistent genre-level pattern: systems perform well on narrative text such as autobiography and play, but degrade sharply on poetry, legal, and conversational text. We further identify a distinct failure mode in poetry translation, where systems reproduce memorized canonical Hindi verse instead of translating the given English text, and manual analysis surfaces recurring issues in named-entity handling, wrong-language output, and lexical choice, several of which persist from our prior CoST evaluation on WMT24 submissions. Our test suite is available at https://github.com/deciphyre/CoST-WMT-26-Test-Suite-Task.</abstract>
<identifier type="citekey">yadav-etal-2026-finding</identifier>
<location>
<url>https://aclanthology.org/2026.wmt-main.93/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1602</start>
<end>1614</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Finding the Gaps in LLM Translation with CoST
%A Yadav, Saumitra
%A Mukherjee, Ananya
%A Shrivastava, Manish
%Y Haddow, Barry
%Y Kocmi, Tom
%Y Koehn, Philipp
%Y Monz, Christof
%S Proceedings of the Eleventh Conference on Machine Translation
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-474-3
%F yadav-etal-2026-finding
%X Large language models (LLMs) have advanced machine translation substantially, but generic benchmarks often fail to expose where these systems struggle with complex linguistic structures and stylistic nuance. We use CoST (Complex Structures Test), a challenge suite of 1,947 English-Hindi sentences spanning eight genres, autobiography, conversation, legal, mixed, narration, play, poetry, and technical writing, previously introduced for WMT24, to evaluate 22 machine translation systems submitted to the WMT26 General Translation Shared Task for English-Hindi with reference-free, reference-based, and manual evaluation. Our results reveal a consistent genre-level pattern: systems perform well on narrative text such as autobiography and play, but degrade sharply on poetry, legal, and conversational text. We further identify a distinct failure mode in poetry translation, where systems reproduce memorized canonical Hindi verse instead of translating the given English text, and manual analysis surfaces recurring issues in named-entity handling, wrong-language output, and lexical choice, several of which persist from our prior CoST evaluation on WMT24 submissions. Our test suite is available at https://github.com/deciphyre/CoST-WMT-26-Test-Suite-Task.
%U https://aclanthology.org/2026.wmt-main.93/
%P 1602-1614
Markdown (Informal)
[Finding the Gaps in LLM Translation with CoST](https://aclanthology.org/2026.wmt-main.93/) (Yadav et al., WMT 2026)
ACL
- Saumitra Yadav, Ananya Mukherjee, and Manish Shrivastava. 2026. Finding the Gaps in LLM Translation with CoST. In Proceedings of the Eleventh Conference on Machine Translation, pages 1602–1614, Budapest, Hungary. Association for Computational Linguistics.