@inproceedings{li-etal-2026-association,
title = "Do Association-Reweighted {BPE} Merge Rules Help Data-Efficient Language Modeling?",
author = "Li, Zy and
Han, Phu and
Tran, Trinh and
Duong, Lam and
Fang, Andrew",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.8/",
pages = "93--104",
ISBN = "979-8-89176-475-0",
abstract = "Subword tokenization determines a language model{'}s symbol inventory and sequence length, yet standard BPE selects merges by frequency alone. We test whether association-reweighted merges help on BabyLM Strict-Small. Holding the merge budget and architecture fixed and approximately matching word exposure, we train 97M-parameter causal decoders with three seeds using standard BPE, the S-BPE merge score with fixed-budget stopping, and exponential positive-PMI reweighting (PMI-BPE). Neither alternative reliably improves held-out bits-per-byte (BPB) or BLiMP. PMI-BPE{'}s mean BPB is slightly higher than standard BPE{'}s, but no pairwise difference is statistically reliable at three seeds. Both alternatives alter the merge tables and measurably change compression, ruling out a trivial no-intervention explanation. Yet intrinsic association rankings depend on scoring measure and aggregation. Type-uniform rankings reverse between post-merge PMI and thresholded held-out PPMI, while application-weighted PPMI is nearly identical across tokenizers. These results do not rule out association-aware tokenization; they cover one merge budget, one operating point per score, and two evaluation axes. Within that scope, the tested scores do not reliably help, and intrinsic merge association does not consistently align with the BPB and BLiMP point-estimate orderings."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-etal-2026-association">
<titleInfo>
<title>Do Association-Reweighted BPE Merge Rules Help Data-Efficient Language Modeling?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Phu</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Trinh</namePart>
<namePart type="family">Tran</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lam</namePart>
<namePart type="family">Duong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andrew</namePart>
<namePart type="family">Fang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>Subword tokenization determines a language model’s symbol inventory and sequence length, yet standard BPE selects merges by frequency alone. We test whether association-reweighted merges help on BabyLM Strict-Small. Holding the merge budget and architecture fixed and approximately matching word exposure, we train 97M-parameter causal decoders with three seeds using standard BPE, the S-BPE merge score with fixed-budget stopping, and exponential positive-PMI reweighting (PMI-BPE). Neither alternative reliably improves held-out bits-per-byte (BPB) or BLiMP. PMI-BPE’s mean BPB is slightly higher than standard BPE’s, but no pairwise difference is statistically reliable at three seeds. Both alternatives alter the merge tables and measurably change compression, ruling out a trivial no-intervention explanation. Yet intrinsic association rankings depend on scoring measure and aggregation. Type-uniform rankings reverse between post-merge PMI and thresholded held-out PPMI, while application-weighted PPMI is nearly identical across tokenizers. These results do not rule out association-aware tokenization; they cover one merge budget, one operating point per score, and two evaluation axes. Within that scope, the tested scores do not reliably help, and intrinsic merge association does not consistently align with the BPB and BLiMP point-estimate orderings.</abstract>
<identifier type="citekey">li-etal-2026-association</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.8/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>93</start>
<end>104</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Do Association-Reweighted BPE Merge Rules Help Data-Efficient Language Modeling?
%A Li, Zy
%A Han, Phu
%A Tran, Trinh
%A Duong, Lam
%A Fang, Andrew
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F li-etal-2026-association
%X Subword tokenization determines a language model’s symbol inventory and sequence length, yet standard BPE selects merges by frequency alone. We test whether association-reweighted merges help on BabyLM Strict-Small. Holding the merge budget and architecture fixed and approximately matching word exposure, we train 97M-parameter causal decoders with three seeds using standard BPE, the S-BPE merge score with fixed-budget stopping, and exponential positive-PMI reweighting (PMI-BPE). Neither alternative reliably improves held-out bits-per-byte (BPB) or BLiMP. PMI-BPE’s mean BPB is slightly higher than standard BPE’s, but no pairwise difference is statistically reliable at three seeds. Both alternatives alter the merge tables and measurably change compression, ruling out a trivial no-intervention explanation. Yet intrinsic association rankings depend on scoring measure and aggregation. Type-uniform rankings reverse between post-merge PMI and thresholded held-out PPMI, while application-weighted PPMI is nearly identical across tokenizers. These results do not rule out association-aware tokenization; they cover one merge budget, one operating point per score, and two evaluation axes. Within that scope, the tested scores do not reliably help, and intrinsic merge association does not consistently align with the BPB and BLiMP point-estimate orderings.
%U https://aclanthology.org/2026.babylm-main.8/
%P 93-104
Markdown (Informal)
[Do Association-Reweighted BPE Merge Rules Help Data-Efficient Language Modeling?](https://aclanthology.org/2026.babylm-main.8/) (Li et al., BabyLM 2026)
ACL