@inproceedings{zhao-etal-2026-regmix,
title = "{R}eg{M}ix-{D}: Dynamic Data Mixing via Proxy Training Trajectories",
author = "Zhao, Kaiyan and
Miao, Zhongtao and
Aizawa, Akiko and
Tsuruoka, Yoshimasa",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.84/",
pages = "1698--1707",
ISBN = "979-8-89176-444-6",
abstract = "Data mixture selection is critical for Large Language Model pretraining. Existing methods such as RegMix select a single static mixture by fitting a regression model on small-scale proxy runs. We propose RegMix-D, a simple extension of RegMix to dynamic mixing. Our key observation is that proxy runs produce not only endpoint losses, but also full loss trajectories, which can be used to further improve data mixture. By training regression model on these trajectories, we can predict optimal mixtures at multiple training stages. RegMix-D supports two deployment modes: an *offline* variant that generates a complete mixture schedule before target training, and an *online* variant that adapts the mixture during training using observed loss. Experiments on 25B tokens of the Pile dataset with a 1B parameter target model show that RegMix-D consistently improves over baselines across 13 downstream tasks, and the improvement carries over to a 7B target model on 100B tokens. RegMix-D also remains proxy-efficient: it surpasses RegMix even with only 128 proxy models (25{\%} of RegMix{'}s proxy compute budget)."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhao-etal-2026-regmix">
<titleInfo>
<title>RegMix-D: Dynamic Data Mixing via Proxy Training Trajectories</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kaiyan</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhongtao</namePart>
<namePart type="family">Miao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akiko</namePart>
<namePart type="family">Aizawa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yoshimasa</namePart>
<namePart type="family">Tsuruoka</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>Data mixture selection is critical for Large Language Model pretraining. Existing methods such as RegMix select a single static mixture by fitting a regression model on small-scale proxy runs. We propose RegMix-D, a simple extension of RegMix to dynamic mixing. Our key observation is that proxy runs produce not only endpoint losses, but also full loss trajectories, which can be used to further improve data mixture. By training regression model on these trajectories, we can predict optimal mixtures at multiple training stages. RegMix-D supports two deployment modes: an *offline* variant that generates a complete mixture schedule before target training, and an *online* variant that adapts the mixture during training using observed loss. Experiments on 25B tokens of the Pile dataset with a 1B parameter target model show that RegMix-D consistently improves over baselines across 13 downstream tasks, and the improvement carries over to a 7B target model on 100B tokens. RegMix-D also remains proxy-efficient: it surpasses RegMix even with only 128 proxy models (25% of RegMix’s proxy compute budget).</abstract>
<identifier type="citekey">zhao-etal-2026-regmix</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.84/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>1698</start>
<end>1707</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T RegMix-D: Dynamic Data Mixing via Proxy Training Trajectories
%A Zhao, Kaiyan
%A Miao, Zhongtao
%A Aizawa, Akiko
%A Tsuruoka, Yoshimasa
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F zhao-etal-2026-regmix
%X Data mixture selection is critical for Large Language Model pretraining. Existing methods such as RegMix select a single static mixture by fitting a regression model on small-scale proxy runs. We propose RegMix-D, a simple extension of RegMix to dynamic mixing. Our key observation is that proxy runs produce not only endpoint losses, but also full loss trajectories, which can be used to further improve data mixture. By training regression model on these trajectories, we can predict optimal mixtures at multiple training stages. RegMix-D supports two deployment modes: an *offline* variant that generates a complete mixture schedule before target training, and an *online* variant that adapts the mixture during training using observed loss. Experiments on 25B tokens of the Pile dataset with a 1B parameter target model show that RegMix-D consistently improves over baselines across 13 downstream tasks, and the improvement carries over to a 7B target model on 100B tokens. RegMix-D also remains proxy-efficient: it surpasses RegMix even with only 128 proxy models (25% of RegMix’s proxy compute budget).
%U https://aclanthology.org/2026.findings-aacl.84/
%P 1698-1707
Markdown (Informal)
[RegMix-D: Dynamic Data Mixing via Proxy Training Trajectories](https://aclanthology.org/2026.findings-aacl.84/) (Zhao et al., Findings 2026)
ACL