@inproceedings{incidelen-etal-2026-cost,
title = "The Cost of Asking in {T}urkish: Prompt Language Effects on Code {LLM}s",
author = "{\.I}ncidelen, Mert and
Kaya, Melike and
{\.I}{\c{s}}bilir, Elif and
Santur, Yunus",
editor = "Adelani, David Ifeoluwa and
Alabi, Jesujoba and
Arnett, Catherine and
Chang, Tyler and
Downey, C.M. and
Goldman, Omer and
Li, Senyu and
Miranda, Lester James Validad and
Qi, Jirui and
Stap, David and
Winata, Genta Indra and
Zhang, Xinyu Crystina",
booktitle = "Proceedings of the 6th Workshop on Multilingual Representation Learning ({MRL} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mrl-1.39/",
pages = "492--496",
ISBN = "979-8-89176-469-9",
abstract = "Large language models (LLMs) have demonstrated strong performance on existing code generation benchmarks. However, these benchmarks are almost entirely based on English prompts. This study examines the cost of prompt language on code generation performance for Turkish, an agglutinative and morphologically rich language. For this purpose, EN-TR CodeBench, a benchmark consisting of 60 problems and 811 unit tests at easy, medium, and hard difficulty levels, was created. Five open-source code LLMs from three model families, one with three different parameter scales, were evaluated under English, Turkish, and Turkish-localized conditions using the pass@1 metric. The results show that the use of Turkish prompts leads to a performance decrease in all five models, and this decrease narrows as model scale increases within the Qwen2.5-Coder family. Analysis according to difficulty level reveals that, for most models, the language-related performance loss increases substantially with problem complexity. Error analysis shows that the performance difference between languages is largely driven by runtime and logic errors, while syntax errors remain rare."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="incidelen-etal-2026-cost">
<titleInfo>
<title>The Cost of Asking in Turkish: Prompt Language Effects on Code LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mert</namePart>
<namePart type="family">İncidelen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Melike</namePart>
<namePart type="family">Kaya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elif</namePart>
<namePart type="family">İşbilir</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yunus</namePart>
<namePart type="family">Santur</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Ifeoluwa</namePart>
<namePart type="family">Adelani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesujoba</namePart>
<namePart type="family">Alabi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Catherine</namePart>
<namePart type="family">Arnett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">C</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Downey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omer</namePart>
<namePart type="family">Goldman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Senyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lester</namePart>
<namePart type="given">James</namePart>
<namePart type="given">Validad</namePart>
<namePart type="family">Miranda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jirui</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Stap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Genta</namePart>
<namePart type="given">Indra</namePart>
<namePart type="family">Winata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="given">Crystina</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-469-9</identifier>
</relatedItem>
<abstract>Large language models (LLMs) have demonstrated strong performance on existing code generation benchmarks. However, these benchmarks are almost entirely based on English prompts. This study examines the cost of prompt language on code generation performance for Turkish, an agglutinative and morphologically rich language. For this purpose, EN-TR CodeBench, a benchmark consisting of 60 problems and 811 unit tests at easy, medium, and hard difficulty levels, was created. Five open-source code LLMs from three model families, one with three different parameter scales, were evaluated under English, Turkish, and Turkish-localized conditions using the pass@1 metric. The results show that the use of Turkish prompts leads to a performance decrease in all five models, and this decrease narrows as model scale increases within the Qwen2.5-Coder family. Analysis according to difficulty level reveals that, for most models, the language-related performance loss increases substantially with problem complexity. Error analysis shows that the performance difference between languages is largely driven by runtime and logic errors, while syntax errors remain rare.</abstract>
<identifier type="citekey">incidelen-etal-2026-cost</identifier>
<location>
<url>https://aclanthology.org/2026.mrl-1.39/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>492</start>
<end>496</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Cost of Asking in Turkish: Prompt Language Effects on Code LLMs
%A İncidelen, Mert
%A Kaya, Melike
%A İşbilir, Elif
%A Santur, Yunus
%Y Adelani, David Ifeoluwa
%Y Alabi, Jesujoba
%Y Arnett, Catherine
%Y Chang, Tyler
%Y Downey, C. M.
%Y Goldman, Omer
%Y Li, Senyu
%Y Miranda, Lester James Validad
%Y Qi, Jirui
%Y Stap, David
%Y Winata, Genta Indra
%Y Zhang, Xinyu Crystina
%S Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-469-9
%F incidelen-etal-2026-cost
%X Large language models (LLMs) have demonstrated strong performance on existing code generation benchmarks. However, these benchmarks are almost entirely based on English prompts. This study examines the cost of prompt language on code generation performance for Turkish, an agglutinative and morphologically rich language. For this purpose, EN-TR CodeBench, a benchmark consisting of 60 problems and 811 unit tests at easy, medium, and hard difficulty levels, was created. Five open-source code LLMs from three model families, one with three different parameter scales, were evaluated under English, Turkish, and Turkish-localized conditions using the pass@1 metric. The results show that the use of Turkish prompts leads to a performance decrease in all five models, and this decrease narrows as model scale increases within the Qwen2.5-Coder family. Analysis according to difficulty level reveals that, for most models, the language-related performance loss increases substantially with problem complexity. Error analysis shows that the performance difference between languages is largely driven by runtime and logic errors, while syntax errors remain rare.
%U https://aclanthology.org/2026.mrl-1.39/
%P 492-496
Markdown (Informal)
[The Cost of Asking in Turkish: Prompt Language Effects on Code LLMs](https://aclanthology.org/2026.mrl-1.39/) (İncidelen et al., MRL 2026)
ACL