@inproceedings{jin-xiong-2026-beyond,
title = "Beyond Aggregate Loss: Training Dynamics and Language-Specific Scaling Laws for Multilingual Language Models",
author = "Jin, Renren and
Xiong, Deyi",
editor = "Adelani, David Ifeoluwa and
Alabi, Jesujoba and
Arnett, Catherine and
Chang, Tyler and
Downey, C.M. and
Goldman, Omer and
Li, Senyu and
Miranda, Lester James Validad and
Qi, Jirui and
Stap, David and
Winata, Genta Indra and
Zhang, Xinyu Crystina",
booktitle = "Proceedings of the 6th Workshop on Multilingual Representation Learning ({MRL} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mrl-1.23/",
pages = "289--299",
ISBN = "979-8-89176-469-9",
abstract = "Scaling laws typically characterize the performance of large language models (LLMs) through aggregate loss, potentially obscuring substantial variation across languages in multilingual pretraining. We study this variation through controlled experiments on a diverse set of languages spanning two language families, training 0.3B- and 0.5B-parameter models on up to 137B tokens. Specifically, we compare continued pretraining from a strong pretrained model with training the same architecture from scratch, examine the effects of expanding the multilingual corpus, analyze cross-family transfer through language-specific loss trajectories, and fit separate scaling laws for individual languages. Our results show that continued pretraining from a strong pretrained model leads to faster convergence and consistently stronger downstream performance than training from scratch. In contrast, reductions in training loss and more stable gradient $\ell_2$ norms do not necessarily translate into downstream benchmark gains. We further demonstrate that training on a single language family can reduce loss for languages in another family, providing evidence of cross-family transfer. Finally, we find that data-scaling behavior is relatively consistent across languages, whereas the benefits of model scaling vary substantially. A sampling-weighted combination of the language-specific scaling laws closely reconstructs the aggregate training loss. These findings show that aggregate metrics provide an incomplete account of multilingual pretraining dynamics and highlight the importance of language-specific analyses for evaluating and scaling multilingual models."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="jin-xiong-2026-beyond">
<titleInfo>
<title>Beyond Aggregate Loss: Training Dynamics and Language-Specific Scaling Laws for Multilingual Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Renren</namePart>
<namePart type="family">Jin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Deyi</namePart>
<namePart type="family">Xiong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Ifeoluwa</namePart>
<namePart type="family">Adelani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesujoba</namePart>
<namePart type="family">Alabi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Catherine</namePart>
<namePart type="family">Arnett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">C</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Downey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Omer</namePart>
<namePart type="family">Goldman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Senyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lester</namePart>
<namePart type="given">James</namePart>
<namePart type="given">Validad</namePart>
<namePart type="family">Miranda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jirui</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Stap</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Genta</namePart>
<namePart type="given">Indra</namePart>
<namePart type="family">Winata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="given">Crystina</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-469-9</identifier>
</relatedItem>
<abstract>Scaling laws typically characterize the performance of large language models (LLMs) through aggregate loss, potentially obscuring substantial variation across languages in multilingual pretraining. We study this variation through controlled experiments on a diverse set of languages spanning two language families, training 0.3B- and 0.5B-parameter models on up to 137B tokens. Specifically, we compare continued pretraining from a strong pretrained model with training the same architecture from scratch, examine the effects of expanding the multilingual corpus, analyze cross-family transfer through language-specific loss trajectories, and fit separate scaling laws for individual languages. Our results show that continued pretraining from a strong pretrained model leads to faster convergence and consistently stronger downstream performance than training from scratch. In contrast, reductions in training loss and more stable gradient \ell₂ norms do not necessarily translate into downstream benchmark gains. We further demonstrate that training on a single language family can reduce loss for languages in another family, providing evidence of cross-family transfer. Finally, we find that data-scaling behavior is relatively consistent across languages, whereas the benefits of model scaling vary substantially. A sampling-weighted combination of the language-specific scaling laws closely reconstructs the aggregate training loss. These findings show that aggregate metrics provide an incomplete account of multilingual pretraining dynamics and highlight the importance of language-specific analyses for evaluating and scaling multilingual models.</abstract>
<identifier type="citekey">jin-xiong-2026-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.mrl-1.23/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>289</start>
<end>299</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Aggregate Loss: Training Dynamics and Language-Specific Scaling Laws for Multilingual Language Models
%A Jin, Renren
%A Xiong, Deyi
%Y Adelani, David Ifeoluwa
%Y Alabi, Jesujoba
%Y Arnett, Catherine
%Y Chang, Tyler
%Y Downey, C. M.
%Y Goldman, Omer
%Y Li, Senyu
%Y Miranda, Lester James Validad
%Y Qi, Jirui
%Y Stap, David
%Y Winata, Genta Indra
%Y Zhang, Xinyu Crystina
%S Proceedings of the 6th Workshop on Multilingual Representation Learning (MRL 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-469-9
%F jin-xiong-2026-beyond
%X Scaling laws typically characterize the performance of large language models (LLMs) through aggregate loss, potentially obscuring substantial variation across languages in multilingual pretraining. We study this variation through controlled experiments on a diverse set of languages spanning two language families, training 0.3B- and 0.5B-parameter models on up to 137B tokens. Specifically, we compare continued pretraining from a strong pretrained model with training the same architecture from scratch, examine the effects of expanding the multilingual corpus, analyze cross-family transfer through language-specific loss trajectories, and fit separate scaling laws for individual languages. Our results show that continued pretraining from a strong pretrained model leads to faster convergence and consistently stronger downstream performance than training from scratch. In contrast, reductions in training loss and more stable gradient \ell₂ norms do not necessarily translate into downstream benchmark gains. We further demonstrate that training on a single language family can reduce loss for languages in another family, providing evidence of cross-family transfer. Finally, we find that data-scaling behavior is relatively consistent across languages, whereas the benefits of model scaling vary substantially. A sampling-weighted combination of the language-specific scaling laws closely reconstructs the aggregate training loss. These findings show that aggregate metrics provide an incomplete account of multilingual pretraining dynamics and highlight the importance of language-specific analyses for evaluating and scaling multilingual models.
%U https://aclanthology.org/2026.mrl-1.23/
%P 289-299
Markdown (Informal)
[Beyond Aggregate Loss: Training Dynamics and Language-Specific Scaling Laws for Multilingual Language Models](https://aclanthology.org/2026.mrl-1.23/) (Jin & Xiong, MRL 2026)
ACL