@inproceedings{wu-etal-2026-pairing,
title = "Pairing Original Text with Meaning-Preserving Rewrites for Sample-Efficient Small-Data Language Model Pretraining",
author = "Wu, Shaoxiang and
Fraser, Alexander and
Edman, Lukas",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.48/",
pages = "611--623",
ISBN = "979-8-89176-475-0",
abstract = "We investigate whether augmenting training data with LLM-generated rewrites can improve performance in small-data language model pretraining. We pair each training sentence with a meaning-preserving rewrite, so that masked tokens can be predicted using both the original and its paired version in the same training window. On FineWeb, we test three rewrite types: *simplification* (shorter and easier), *explanation* (longer and easier), and *paraphrasing* (similar length but harder to read). We also test simplification on OpenStax+BCCampus and Simple Wikipedia. We add a reverse masking curriculum: 7 epochs of whole-word masking (WWM), where entire words are masked instead of individual subword tokens, followed by 3 epochs of standard token-level masking (WWM7-to-Token3). The masking procedure is not rewrite-aware: it does not use alignments between the original sentence and its rewrite. All three FineWeb rewrite types improve EWoK and GLUE-style fine-tuning over source-matched baselines, and simplification improves Entity Tracking across all three sources. The EWoK and GLUE gains concentrate on concept-swap and sentence-pair tasks, suggesting that rewrite pairing especially helps models link meaning across different surface forms. Despite using only one-tenth as much training data, our best model surpasses the official BabyLM 100M GPT-2 baseline on EWoK, Entity Tracking, and GLUE, suggesting that paired rewrite augmentation can improve sample efficiency under a fixed small-data budget."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wu-etal-2026-pairing">
<titleInfo>
<title>Pairing Original Text with Meaning-Preserving Rewrites for Sample-Efficient Small-Data Language Model Pretraining</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shaoxiang</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexander</namePart>
<namePart type="family">Fraser</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lukas</namePart>
<namePart type="family">Edman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>We investigate whether augmenting training data with LLM-generated rewrites can improve performance in small-data language model pretraining. We pair each training sentence with a meaning-preserving rewrite, so that masked tokens can be predicted using both the original and its paired version in the same training window. On FineWeb, we test three rewrite types: *simplification* (shorter and easier), *explanation* (longer and easier), and *paraphrasing* (similar length but harder to read). We also test simplification on OpenStax+BCCampus and Simple Wikipedia. We add a reverse masking curriculum: 7 epochs of whole-word masking (WWM), where entire words are masked instead of individual subword tokens, followed by 3 epochs of standard token-level masking (WWM7-to-Token3). The masking procedure is not rewrite-aware: it does not use alignments between the original sentence and its rewrite. All three FineWeb rewrite types improve EWoK and GLUE-style fine-tuning over source-matched baselines, and simplification improves Entity Tracking across all three sources. The EWoK and GLUE gains concentrate on concept-swap and sentence-pair tasks, suggesting that rewrite pairing especially helps models link meaning across different surface forms. Despite using only one-tenth as much training data, our best model surpasses the official BabyLM 100M GPT-2 baseline on EWoK, Entity Tracking, and GLUE, suggesting that paired rewrite augmentation can improve sample efficiency under a fixed small-data budget.</abstract>
<identifier type="citekey">wu-etal-2026-pairing</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.48/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>611</start>
<end>623</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Pairing Original Text with Meaning-Preserving Rewrites for Sample-Efficient Small-Data Language Model Pretraining
%A Wu, Shaoxiang
%A Fraser, Alexander
%A Edman, Lukas
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F wu-etal-2026-pairing
%X We investigate whether augmenting training data with LLM-generated rewrites can improve performance in small-data language model pretraining. We pair each training sentence with a meaning-preserving rewrite, so that masked tokens can be predicted using both the original and its paired version in the same training window. On FineWeb, we test three rewrite types: *simplification* (shorter and easier), *explanation* (longer and easier), and *paraphrasing* (similar length but harder to read). We also test simplification on OpenStax+BCCampus and Simple Wikipedia. We add a reverse masking curriculum: 7 epochs of whole-word masking (WWM), where entire words are masked instead of individual subword tokens, followed by 3 epochs of standard token-level masking (WWM7-to-Token3). The masking procedure is not rewrite-aware: it does not use alignments between the original sentence and its rewrite. All three FineWeb rewrite types improve EWoK and GLUE-style fine-tuning over source-matched baselines, and simplification improves Entity Tracking across all three sources. The EWoK and GLUE gains concentrate on concept-swap and sentence-pair tasks, suggesting that rewrite pairing especially helps models link meaning across different surface forms. Despite using only one-tenth as much training data, our best model surpasses the official BabyLM 100M GPT-2 baseline on EWoK, Entity Tracking, and GLUE, suggesting that paired rewrite augmentation can improve sample efficiency under a fixed small-data budget.
%U https://aclanthology.org/2026.babylm-main.48/
%P 611-623
Markdown (Informal)
[Pairing Original Text with Meaning-Preserving Rewrites for Sample-Efficient Small-Data Language Model Pretraining](https://aclanthology.org/2026.babylm-main.48/) (Wu et al., BabyLM 2026)
ACL