@inproceedings{kim-2026-rethinking,
title = "Rethinking Gradient Flow Through Frozen Blocks for Memory-Efficient Block-Coordinate Training",
author = "Kim, Yeachan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1387/",
pages = "31675--31685",
ISBN = "979-8-89176-459-0",
abstract = "Block-coordinate descent (BCD) has emerged as an effective approach for memory-efficient fine-tuning of large language models. During training, BCD randomly activates a small subset of blocks, such as layers or modules, and updates only those blocks while keeping the rest frozen. Although effective, prior work has paid little attention to the frozen blocks, which must still retain forward activations to propagate gradients during the backward pass. Our analysis reveals that gradients through these frozen blocks are highly sparse at the token level: a small subset of tokens carries nearly all of the signal required by the active blocks. Building on this observation, we introduce frozen-block gradient approximation with sparse tokens (Frost), which approximates gradients through inactive blocks using only a sparse subset of tokens and integrates seamlessly with standard BCD optimizers. Across diverse fine-tuning scenarios, model backbones, and modalities, Frost substantially improves the memory efficiency of BCD while preserving downstream performance with negligible degradation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kim-2026-rethinking">
<titleInfo>
<title>Rethinking Gradient Flow Through Frozen Blocks for Memory-Efficient Block-Coordinate Training</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yeachan</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Block-coordinate descent (BCD) has emerged as an effective approach for memory-efficient fine-tuning of large language models. During training, BCD randomly activates a small subset of blocks, such as layers or modules, and updates only those blocks while keeping the rest frozen. Although effective, prior work has paid little attention to the frozen blocks, which must still retain forward activations to propagate gradients during the backward pass. Our analysis reveals that gradients through these frozen blocks are highly sparse at the token level: a small subset of tokens carries nearly all of the signal required by the active blocks. Building on this observation, we introduce frozen-block gradient approximation with sparse tokens (Frost), which approximates gradients through inactive blocks using only a sparse subset of tokens and integrates seamlessly with standard BCD optimizers. Across diverse fine-tuning scenarios, model backbones, and modalities, Frost substantially improves the memory efficiency of BCD while preserving downstream performance with negligible degradation.</abstract>
<identifier type="citekey">kim-2026-rethinking</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1387/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>31675</start>
<end>31685</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Rethinking Gradient Flow Through Frozen Blocks for Memory-Efficient Block-Coordinate Training
%A Kim, Yeachan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F kim-2026-rethinking
%X Block-coordinate descent (BCD) has emerged as an effective approach for memory-efficient fine-tuning of large language models. During training, BCD randomly activates a small subset of blocks, such as layers or modules, and updates only those blocks while keeping the rest frozen. Although effective, prior work has paid little attention to the frozen blocks, which must still retain forward activations to propagate gradients during the backward pass. Our analysis reveals that gradients through these frozen blocks are highly sparse at the token level: a small subset of tokens carries nearly all of the signal required by the active blocks. Building on this observation, we introduce frozen-block gradient approximation with sparse tokens (Frost), which approximates gradients through inactive blocks using only a sparse subset of tokens and integrates seamlessly with standard BCD optimizers. Across diverse fine-tuning scenarios, model backbones, and modalities, Frost substantially improves the memory efficiency of BCD while preserving downstream performance with negligible degradation.
%U https://aclanthology.org/2026.emnlp-main.1387/
%P 31675-31685
Markdown (Informal)
[Rethinking Gradient Flow Through Frozen Blocks for Memory-Efficient Block-Coordinate Training](https://aclanthology.org/2026.emnlp-main.1387/) (Kim, EMNLP 2026)
ACL