@inproceedings{vasileva-2026-im,
title = "``{I}{'}m Sorry, {I} Can{'}t Translate That'': A Challenge Set for Refusal to Translate, Overgeneration and Hallucination in {LLM} Translation Systems",
author = "Vasileva, Lisa",
editor = "Haddow, Barry and
Kocmi, Tom and
Koehn, Philipp and
Monz, Christof",
booktitle = "Proceedings of the Eleventh Conference on Machine Translation",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.wmt-main.92/",
pages = "1594--1601",
ISBN = "979-8-89176-474-3",
abstract = "Large language models (LLMs) have significantly raised the bar for translation quality in the recent years. However, they are still prone to novel failure modes, including hallucinations and overgenerations, where the generated translation is partially or fully detached from the source text and contains information that is not present in the input. To facilitate the analysis of these behaviors and assess the robustness of LLM-based translation systems, we introduce a challenge set specifically designed to probe scenarios that we have observed to be particularly prone to overgeneration, hallucination, and other forms of semantic detachment from the source. We describe the construction of the challenge set and present the evaluation of the systems submitted to the WMT 2026 General Machine Translation shared task."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="vasileva-2026-im">
<titleInfo>
<title>“I’m Sorry, I Can’t Translate That”: A Challenge Set for Refusal to Translate, Overgeneration and Hallucination in LLM Translation Systems</title>
</titleInfo>
<name type="personal">
<namePart type="given">Lisa</namePart>
<namePart type="family">Vasileva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Eleventh Conference on Machine Translation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Barry</namePart>
<namePart type="family">Haddow</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tom</namePart>
<namePart type="family">Kocmi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Philipp</namePart>
<namePart type="family">Koehn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christof</namePart>
<namePart type="family">Monz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-474-3</identifier>
</relatedItem>
<abstract>Large language models (LLMs) have significantly raised the bar for translation quality in the recent years. However, they are still prone to novel failure modes, including hallucinations and overgenerations, where the generated translation is partially or fully detached from the source text and contains information that is not present in the input. To facilitate the analysis of these behaviors and assess the robustness of LLM-based translation systems, we introduce a challenge set specifically designed to probe scenarios that we have observed to be particularly prone to overgeneration, hallucination, and other forms of semantic detachment from the source. We describe the construction of the challenge set and present the evaluation of the systems submitted to the WMT 2026 General Machine Translation shared task.</abstract>
<identifier type="citekey">vasileva-2026-im</identifier>
<location>
<url>https://aclanthology.org/2026.wmt-main.92/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1594</start>
<end>1601</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T “I’m Sorry, I Can’t Translate That”: A Challenge Set for Refusal to Translate, Overgeneration and Hallucination in LLM Translation Systems
%A Vasileva, Lisa
%Y Haddow, Barry
%Y Kocmi, Tom
%Y Koehn, Philipp
%Y Monz, Christof
%S Proceedings of the Eleventh Conference on Machine Translation
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-474-3
%F vasileva-2026-im
%X Large language models (LLMs) have significantly raised the bar for translation quality in the recent years. However, they are still prone to novel failure modes, including hallucinations and overgenerations, where the generated translation is partially or fully detached from the source text and contains information that is not present in the input. To facilitate the analysis of these behaviors and assess the robustness of LLM-based translation systems, we introduce a challenge set specifically designed to probe scenarios that we have observed to be particularly prone to overgeneration, hallucination, and other forms of semantic detachment from the source. We describe the construction of the challenge set and present the evaluation of the systems submitted to the WMT 2026 General Machine Translation shared task.
%U https://aclanthology.org/2026.wmt-main.92/
%P 1594-1601
Markdown (Informal)
["I’m Sorry, I Can’t Translate That": A Challenge Set for Refusal to Translate, Overgeneration and Hallucination in LLM Translation Systems](https://aclanthology.org/2026.wmt-main.92/) (Vasileva, WMT 2026)
ACL