@inproceedings{lianhai-gong-2026-compressing,
title = "Compressing Only What Is Compressible: Memory-Efficient Multiscale Muon via Selective Low-Rank Projection",
author = "Lianhai, Ren and
Gong, Yeyun",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.461/",
pages = "9675--9692",
ISBN = "979-8-89176-462-0",
abstract = "Multiscale Muon maintains $K$ momentum EMAs at different decay rates $\beta_k$, doubling or tripling the optimizer-state memory of Muon. We show that the EMA update is a first-order low-pass filter and Multiscale Muon is therefore a \textit{filter bank} on the gradient stream, with each band tuned to a different temporal scale. Under this view, the empirical fact that Multiscale Muon outperforms any single-$\beta$ filter is a direct consequence of multi-band coverage; and a sharp, asymmetric compressibility property follows from the bandwidth{---}the narrow-band (high-$\beta$) buffer is dominated by the slow, low-rank component of the gradient and is safe to project, while the wide-band (low-$\beta$) buffer carries high-frequency content whose effective rank is large and cannot be compressed without hurting convergence. We formalize this asymmetry, instantiate it in SMuG (Selective Multiscale Muon with GaLore) by applying GaLore only to high-$\beta$ buffers, and prove an algebraic impossibility result ruling out any non-negative linear spectral score from predicting the band-mixing coefficient $\lambda^\star$ in closed form. On 200,M GPT-2 pretraining ($n{=}4$ seeds), SMuG ($K{=}2$, $r/n{=}0.125$) reaches PPL $21.66{\pm}0.05$, statistically tied with full-rank Multiscale Muon ($21.69{\pm}0.03$), at $1.25\times$ Muon{'}s optimizer memory (including the persistent GaLore basis $V$) versus $2.00\times$ for full Multiscale Muon{---}recovering 75{\%} of the added overhead at no average quality cost."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lianhai-gong-2026-compressing">
<titleInfo>
<title>Compressing Only What Is Compressible: Memory-Efficient Multiscale Muon via Selective Low-Rank Projection</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ren</namePart>
<namePart type="family">Lianhai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yeyun</namePart>
<namePart type="family">Gong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Multiscale Muon maintains K momentum EMAs at different decay rates β_k, doubling or tripling the optimizer-state memory of Muon. We show that the EMA update is a first-order low-pass filter and Multiscale Muon is therefore a filter bank on the gradient stream, with each band tuned to a different temporal scale. Under this view, the empirical fact that Multiscale Muon outperforms any single-β filter is a direct consequence of multi-band coverage; and a sharp, asymmetric compressibility property follows from the bandwidth—the narrow-band (high-β) buffer is dominated by the slow, low-rank component of the gradient and is safe to project, while the wide-band (low-β) buffer carries high-frequency content whose effective rank is large and cannot be compressed without hurting convergence. We formalize this asymmetry, instantiate it in SMuG (Selective Multiscale Muon with GaLore) by applying GaLore only to high-β buffers, and prove an algebraic impossibility result ruling out any non-negative linear spectral score from predicting the band-mixing coefficient łambda^\star in closed form. On 200,M GPT-2 pretraining (n=4 seeds), SMuG (K=2, r/n=0.125) reaches PPL 21.66\pm0.05, statistically tied with full-rank Multiscale Muon (21.69\pm0.03), at 1.25\times Muon’s optimizer memory (including the persistent GaLore basis V) versus 2.00\times for full Multiscale Muon—recovering 75% of the added overhead at no average quality cost.</abstract>
<identifier type="citekey">lianhai-gong-2026-compressing</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.461/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>9675</start>
<end>9692</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Compressing Only What Is Compressible: Memory-Efficient Multiscale Muon via Selective Low-Rank Projection
%A Lianhai, Ren
%A Gong, Yeyun
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F lianhai-gong-2026-compressing
%X Multiscale Muon maintains K momentum EMAs at different decay rates β_k, doubling or tripling the optimizer-state memory of Muon. We show that the EMA update is a first-order low-pass filter and Multiscale Muon is therefore a filter bank on the gradient stream, with each band tuned to a different temporal scale. Under this view, the empirical fact that Multiscale Muon outperforms any single-β filter is a direct consequence of multi-band coverage; and a sharp, asymmetric compressibility property follows from the bandwidth—the narrow-band (high-β) buffer is dominated by the slow, low-rank component of the gradient and is safe to project, while the wide-band (low-β) buffer carries high-frequency content whose effective rank is large and cannot be compressed without hurting convergence. We formalize this asymmetry, instantiate it in SMuG (Selective Multiscale Muon with GaLore) by applying GaLore only to high-β buffers, and prove an algebraic impossibility result ruling out any non-negative linear spectral score from predicting the band-mixing coefficient łambda^\star in closed form. On 200,M GPT-2 pretraining (n=4 seeds), SMuG (K=2, r/n=0.125) reaches PPL 21.66\pm0.05, statistically tied with full-rank Multiscale Muon (21.69\pm0.03), at 1.25\times Muon’s optimizer memory (including the persistent GaLore basis V) versus 2.00\times for full Multiscale Muon—recovering 75% of the added overhead at no average quality cost.
%U https://aclanthology.org/2026.findings-emnlp.461/
%P 9675-9692
Markdown (Informal)
[Compressing Only What Is Compressible: Memory-Efficient Multiscale Muon via Selective Low-Rank Projection](https://aclanthology.org/2026.findings-emnlp.461/) (Lianhai & Gong, Findings 2026)
ACL