@inproceedings{paudel-sheshappanavar-2026-parameter,
title = "Parameter Golf: What Really Works?",
author = "Paudel, Prashanna Mani and
Sheshappanavar, Shivanand Venkanna",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.11/",
pages = "134--147",
ISBN = "979-8-89176-475-0",
abstract = "How far can a language model improve under a strict artifact budget? Parameter Golf posed this question as an open community challenge in which participants trained the best language model, with the complete artifact (training code + compressed weights) required to fit within 16 MB and to be trained in under ten minutes on 8xH100 SXM GPUs. Quality was measured in bits-per-byte (BPB), the average number of bits required to encode each byte of unseen text. We analyze 2,037 pull requests and 1,430 clean-scored submissions from the contest, build a taxonomy of 84 optimization techniques, and measure each technique{'}s contribution to BPB. The verified leaderboard score dropped from 1.2244 to 1.058 BPB across three phases, a 13.6{\%} reduction, despite individual techniques rarely improving BPB by more than 1{\%}. We show that most techniques' gains shrink when re-measured among competitive submissions, isolating the few methods that help regardless of the surrounding stack. Code and data are available at https://github.com/PMP56/pmgolf-analysis."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="paudel-sheshappanavar-2026-parameter">
<titleInfo>
<title>Parameter Golf: What Really Works?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Prashanna</namePart>
<namePart type="given">Mani</namePart>
<namePart type="family">Paudel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shivanand</namePart>
<namePart type="given">Venkanna</namePart>
<namePart type="family">Sheshappanavar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>How far can a language model improve under a strict artifact budget? Parameter Golf posed this question as an open community challenge in which participants trained the best language model, with the complete artifact (training code + compressed weights) required to fit within 16 MB and to be trained in under ten minutes on 8xH100 SXM GPUs. Quality was measured in bits-per-byte (BPB), the average number of bits required to encode each byte of unseen text. We analyze 2,037 pull requests and 1,430 clean-scored submissions from the contest, build a taxonomy of 84 optimization techniques, and measure each technique’s contribution to BPB. The verified leaderboard score dropped from 1.2244 to 1.058 BPB across three phases, a 13.6% reduction, despite individual techniques rarely improving BPB by more than 1%. We show that most techniques’ gains shrink when re-measured among competitive submissions, isolating the few methods that help regardless of the surrounding stack. Code and data are available at https://github.com/PMP56/pmgolf-analysis.</abstract>
<identifier type="citekey">paudel-sheshappanavar-2026-parameter</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.11/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>134</start>
<end>147</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Parameter Golf: What Really Works?
%A Paudel, Prashanna Mani
%A Sheshappanavar, Shivanand Venkanna
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F paudel-sheshappanavar-2026-parameter
%X How far can a language model improve under a strict artifact budget? Parameter Golf posed this question as an open community challenge in which participants trained the best language model, with the complete artifact (training code + compressed weights) required to fit within 16 MB and to be trained in under ten minutes on 8xH100 SXM GPUs. Quality was measured in bits-per-byte (BPB), the average number of bits required to encode each byte of unseen text. We analyze 2,037 pull requests and 1,430 clean-scored submissions from the contest, build a taxonomy of 84 optimization techniques, and measure each technique’s contribution to BPB. The verified leaderboard score dropped from 1.2244 to 1.058 BPB across three phases, a 13.6% reduction, despite individual techniques rarely improving BPB by more than 1%. We show that most techniques’ gains shrink when re-measured among competitive submissions, isolating the few methods that help regardless of the surrounding stack. Code and data are available at https://github.com/PMP56/pmgolf-analysis.
%U https://aclanthology.org/2026.babylm-main.11/
%P 134-147
Markdown (Informal)
[Parameter Golf: What Really Works?](https://aclanthology.org/2026.babylm-main.11/) (Paudel & Sheshappanavar, BabyLM 2026)
ACL
- Prashanna Mani Paudel and Shivanand Venkanna Sheshappanavar. 2026. Parameter Golf: What Really Works?. In Proceedings of the Second BabyLM Workshop, pages 134–147, Budapest, Hungary. Association for Computational Linguistics.