@inproceedings{yilmaz-etal-2026-impact,
title = "Impact of Bilingual Corpus on Pretraining Large Language Models: An Analysis for {T}urkish",
author = {Y{\i}lmaz, Elif {\"O}zge and
Uysal, Melike and
Toraman, Cagri},
editor = "Adelani, David Ifeoluwa and
Alabi, Jesujoba and
Arnett, Catherine and
Chang, Tyler and
Downey, C.M. and
Goldman, Omer and
Li, Senyu and
Miranda, Lester James Validad and
Qi, Jirui and
Stap, David and
Winata, Genta Indra and
Zhang, Xinyu Crystina",
booktitle = "Proceedings of the 6th Workshop on Multilingual Representation Learning ({MRL} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mrl-1.37/",
pages = "467--479",
ISBN = "979-8-89176-469-9",
abstract = "The development of Large Language Models has primarily focused on English-centric or massively multilingual paradigms. As a result, the dynamics of mixing just two languages from the very beginning of training remain underexplored, especially for smaller models with limited capacity. In this study, we conduct a controlled set of experiments to evaluate the impact of bilingual corpus during from-scratch pretraining. Using a capacity-constrained Gemma-4-derived 181M-parameter architecture trained on a 10 billion token budget, we compare a purely Turkish baseline against mixtures containing 10{\%}, 20{\%}, or 30{\%} of an auxiliary language. We test four languages to isolate different linguistic relationships: English (a global bridge language), Azerbaijani (closely related sibling), Finnish (morphological peer), and French (lexical pivot). Our findings complicate the curse of multilinguality, the tendency for per-language performance to drop as more languages are added. We show that adding any secondary language hurts fluency on structurally uniform text. However, co-training with a typological sibling, Azerbaijani, cuts perplexity on creative and literary text by up to 45{\%} and 51{\%} respectively relative to the monolingual baseline. At higher mixture ratios, Azerbaijani outperforms other languages including morphological and lexical alternatives."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yilmaz-etal-2026-impact">
<titleInfo>
<title>Impact of Bilingual Corpus on Pretraining Large Language Models: An Analysis for Turkish</title>
</titleInfo>
<name type="personal">
<namePart type="given">Elif</namePart>
<namePart type="given">Özge</namePart>
<namePart type="family">Yılmaz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Melike</namePart>
<namePart type="family">Uysal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cagri</namePart>
<namePart type="family">Toraman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Ifeoluwa</namePart>
<namePart type="family">Adelani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesujoba</namePart>
<namePart type="family">Alabi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Catherine</namePart>
<namePart type="family">Arnett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">C</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Downey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omer</namePart>
<namePart type="family">Goldman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Senyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lester</namePart>
<namePart type="given">James</namePart>
<namePart type="given">Validad</namePart>
<namePart type="family">Miranda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jirui</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Stap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Genta</namePart>
<namePart type="given">Indra</namePart>
<namePart type="family">Winata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="given">Crystina</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-469-9</identifier>
</relatedItem>
<abstract>The development of Large Language Models has primarily focused on English-centric or massively multilingual paradigms. As a result, the dynamics of mixing just two languages from the very beginning of training remain underexplored, especially for smaller models with limited capacity. In this study, we conduct a controlled set of experiments to evaluate the impact of bilingual corpus during from-scratch pretraining. Using a capacity-constrained Gemma-4-derived 181M-parameter architecture trained on a 10 billion token budget, we compare a purely Turkish baseline against mixtures containing 10%, 20%, or 30% of an auxiliary language. We test four languages to isolate different linguistic relationships: English (a global bridge language), Azerbaijani (closely related sibling), Finnish (morphological peer), and French (lexical pivot). Our findings complicate the curse of multilinguality, the tendency for per-language performance to drop as more languages are added. We show that adding any secondary language hurts fluency on structurally uniform text. However, co-training with a typological sibling, Azerbaijani, cuts perplexity on creative and literary text by up to 45% and 51% respectively relative to the monolingual baseline. At higher mixture ratios, Azerbaijani outperforms other languages including morphological and lexical alternatives.</abstract>
<identifier type="citekey">yilmaz-etal-2026-impact</identifier>
<location>
<url>https://aclanthology.org/2026.mrl-1.37/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>467</start>
<end>479</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Impact of Bilingual Corpus on Pretraining Large Language Models: An Analysis for Turkish
%A Yılmaz, Elif Özge
%A Uysal, Melike
%A Toraman, Cagri
%Y Adelani, David Ifeoluwa
%Y Alabi, Jesujoba
%Y Arnett, Catherine
%Y Chang, Tyler
%Y Downey, C. M.
%Y Goldman, Omer
%Y Li, Senyu
%Y Miranda, Lester James Validad
%Y Qi, Jirui
%Y Stap, David
%Y Winata, Genta Indra
%Y Zhang, Xinyu Crystina
%S Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-469-9
%F yilmaz-etal-2026-impact
%X The development of Large Language Models has primarily focused on English-centric or massively multilingual paradigms. As a result, the dynamics of mixing just two languages from the very beginning of training remain underexplored, especially for smaller models with limited capacity. In this study, we conduct a controlled set of experiments to evaluate the impact of bilingual corpus during from-scratch pretraining. Using a capacity-constrained Gemma-4-derived 181M-parameter architecture trained on a 10 billion token budget, we compare a purely Turkish baseline against mixtures containing 10%, 20%, or 30% of an auxiliary language. We test four languages to isolate different linguistic relationships: English (a global bridge language), Azerbaijani (closely related sibling), Finnish (morphological peer), and French (lexical pivot). Our findings complicate the curse of multilinguality, the tendency for per-language performance to drop as more languages are added. We show that adding any secondary language hurts fluency on structurally uniform text. However, co-training with a typological sibling, Azerbaijani, cuts perplexity on creative and literary text by up to 45% and 51% respectively relative to the monolingual baseline. At higher mixture ratios, Azerbaijani outperforms other languages including morphological and lexical alternatives.
%U https://aclanthology.org/2026.mrl-1.37/
%P 467-479
Markdown (Informal)
[Impact of Bilingual Corpus on Pretraining Large Language Models: An Analysis for Turkish](https://aclanthology.org/2026.mrl-1.37/) (Yılmaz et al., MRL 2026)
ACL