@inproceedings{zheng-bloem-2026-benchmarking,
title = "Benchmarking Multilingual {LLM} Translation Accuracy for Fuzhounese",
author = "Zheng, Sue and
Bloem, Jelke",
editor = "Ojha, Atul Kr. and
Sakti, Sakriani and
Soria, Claudia and
Melero, Maite and
McCrae, John P. and
Lignos, Constantine and
Liu, Chao-Hong and
Claramunt, German Rigau and
Rehm, Georg",
booktitle = "Proceedings of the {SIGUL} 2026 Joint Workshop with {ELE}, {EURALI}, and {DCLRL}: Towards Inclusivity and Equality: Language Resources and Technologies for Under-Resourced and Endangered Languages",
month = may,
year = "2026",
address = "Palma, Mallorca, Spain",
publisher = "ELRA Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.sigul-1.20/",
doi = "10.63317/4mm9bs8yy4ie",
pages = "198--209",
abstract = "Multilingual large language models are known to perform very well on high-resource languages, while their ability to process severely under-resourced languages remains underexplored. We investigate multilingual LLM translation performance on Fuzhounese, an under-resourced Sinitic language without a standardized orthography and almost no digital presence. Having adopted some methodological insights from the HKCanto-Eval benchmark, this paper presents a bidirectional translation framework based on a dataset of 305 sentences (300 constructed English sentences and 5 additional reference translations), that assesses the comprehension and generation of Fuzhounese, evaluated using automatic metrics and human Likert-scale judgments. The results reveal poor performance on Fuzhounese in both translation directions: BERTScore and chrF++ values consistently stay low when models are faced with comprehension tasks, while for generation tasks, scores are generally more than twofold lower than those for Mandarin or Cantonese. These findings highlight structural biases in multilingual LLMs toward high-resource languages and stress the need for resource-aware modeling and evaluation approaches in multilingual NLP systems."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zheng-bloem-2026-benchmarking">
<titleInfo>
<title>Benchmarking Multilingual LLM Translation Accuracy for Fuzhounese</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sue</namePart>
<namePart type="family">Zheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jelke</namePart>
<namePart type="family">Bloem</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the SIGUL 2026 Joint Workshop with ELE, EURALI, and DCLRL: Towards Inclusivity and Equality: Language Resources and Technologies for Under-Resourced and Endangered Languages</title>
</titleInfo>
<name type="personal">
<namePart type="given">Atul</namePart>
<namePart type="given">Kr.</namePart>
<namePart type="family">Ojha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sakriani</namePart>
<namePart type="family">Sakti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Claudia</namePart>
<namePart type="family">Soria</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maite</namePart>
<namePart type="family">Melero</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">John</namePart>
<namePart type="given">P</namePart>
<namePart type="family">McCrae</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Constantine</namePart>
<namePart type="family">Lignos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chao-Hong</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">German</namePart>
<namePart type="given">Rigau</namePart>
<namePart type="family">Claramunt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Georg</namePart>
<namePart type="family">Rehm</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma, Mallorca, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Multilingual large language models are known to perform very well on high-resource languages, while their ability to process severely under-resourced languages remains underexplored. We investigate multilingual LLM translation performance on Fuzhounese, an under-resourced Sinitic language without a standardized orthography and almost no digital presence. Having adopted some methodological insights from the HKCanto-Eval benchmark, this paper presents a bidirectional translation framework based on a dataset of 305 sentences (300 constructed English sentences and 5 additional reference translations), that assesses the comprehension and generation of Fuzhounese, evaluated using automatic metrics and human Likert-scale judgments. The results reveal poor performance on Fuzhounese in both translation directions: BERTScore and chrF++ values consistently stay low when models are faced with comprehension tasks, while for generation tasks, scores are generally more than twofold lower than those for Mandarin or Cantonese. These findings highlight structural biases in multilingual LLMs toward high-resource languages and stress the need for resource-aware modeling and evaluation approaches in multilingual NLP systems.</abstract>
<identifier type="citekey">zheng-bloem-2026-benchmarking</identifier>
<identifier type="doi">10.63317/4mm9bs8yy4ie</identifier>
<location>
<url>https://aclanthology.org/2026.sigul-1.20/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>198</start>
<end>209</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Benchmarking Multilingual LLM Translation Accuracy for Fuzhounese
%A Zheng, Sue
%A Bloem, Jelke
%Y Ojha, Atul Kr.
%Y Sakti, Sakriani
%Y Soria, Claudia
%Y Melero, Maite
%Y McCrae, John P.
%Y Lignos, Constantine
%Y Liu, Chao-Hong
%Y Claramunt, German Rigau
%Y Rehm, Georg
%S Proceedings of the SIGUL 2026 Joint Workshop with ELE, EURALI, and DCLRL: Towards Inclusivity and Equality: Language Resources and Technologies for Under-Resourced and Endangered Languages
%D 2026
%8 May
%I ELRA Language Resources Association (ELRA)
%C Palma, Mallorca, Spain
%F zheng-bloem-2026-benchmarking
%X Multilingual large language models are known to perform very well on high-resource languages, while their ability to process severely under-resourced languages remains underexplored. We investigate multilingual LLM translation performance on Fuzhounese, an under-resourced Sinitic language without a standardized orthography and almost no digital presence. Having adopted some methodological insights from the HKCanto-Eval benchmark, this paper presents a bidirectional translation framework based on a dataset of 305 sentences (300 constructed English sentences and 5 additional reference translations), that assesses the comprehension and generation of Fuzhounese, evaluated using automatic metrics and human Likert-scale judgments. The results reveal poor performance on Fuzhounese in both translation directions: BERTScore and chrF++ values consistently stay low when models are faced with comprehension tasks, while for generation tasks, scores are generally more than twofold lower than those for Mandarin or Cantonese. These findings highlight structural biases in multilingual LLMs toward high-resource languages and stress the need for resource-aware modeling and evaluation approaches in multilingual NLP systems.
%R 10.63317/4mm9bs8yy4ie
%U https://aclanthology.org/2026.sigul-1.20/
%U https://doi.org/10.63317/4mm9bs8yy4ie
%P 198-209
Markdown (Informal)
[Benchmarking Multilingual LLM Translation Accuracy for Fuzhounese](https://aclanthology.org/2026.sigul-1.20/) (Zheng & Bloem, SIGUL-EURALI-DCLRL 2026)
ACL
- Sue Zheng and Jelke Bloem. 2026. Benchmarking Multilingual LLM Translation Accuracy for Fuzhounese. In Proceedings of the SIGUL 2026 Joint Workshop with ELE, EURALI, and DCLRL: Towards Inclusivity and Equality: Language Resources and Technologies for Under-Resourced and Endangered Languages, pages 198–209, Palma, Mallorca, Spain. ELRA Language Resources Association (ELRA).