@inproceedings{kwon-etal-2026-lora,
title = "{L}o{RA}-{RG}: Residual Gradient-Guided Rank Initialization and Expansion",
author = "Kwon, Dayoung and
Lee, Jaeseong and
Hwang, Seung-Won",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2000/",
pages = "45571--45587",
ISBN = "979-8-89176-459-0",
abstract = "We study how to optimally allocate ranks in Low-Rank Adaptation (LoRA). Despite its importance, ranks are typically chosen manually and fixed across all modules. We argue that this practice is suboptimal, risking either under-parameterization, which hurts performance, or over-parameterization, which wastes memory and computation. To address this, we propose LoRA-RG, a gradient-guided framework that adjusts LoRA ranks through two stages: rank initialization and rank expansion. In the rank initialization stage, LoRA-RG estimates a minimal rank for each layer based on gradient information prior to training. During training, the rank expansion stage further increases ranks only when the current LoRA parameterization is insufficient to capture the remaining update demand. This design dynamically aligns LoRA ranks with the underlying optimization needs. We evaluate LoRA-RG on reasoning, mathematical, and code generation benchmarks, and show that it consistently achieves higher accuracy compared to fixed-rank LoRA and recent rank initialization and adaptation baselines under the same parameter budget."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kwon-etal-2026-lora">
<titleInfo>
<title>LoRA-RG: Residual Gradient-Guided Rank Initialization and Expansion</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dayoung</namePart>
<namePart type="family">Kwon</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaeseong</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Seung-Won</namePart>
<namePart type="family">Hwang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>We study how to optimally allocate ranks in Low-Rank Adaptation (LoRA). Despite its importance, ranks are typically chosen manually and fixed across all modules. We argue that this practice is suboptimal, risking either under-parameterization, which hurts performance, or over-parameterization, which wastes memory and computation. To address this, we propose LoRA-RG, a gradient-guided framework that adjusts LoRA ranks through two stages: rank initialization and rank expansion. In the rank initialization stage, LoRA-RG estimates a minimal rank for each layer based on gradient information prior to training. During training, the rank expansion stage further increases ranks only when the current LoRA parameterization is insufficient to capture the remaining update demand. This design dynamically aligns LoRA ranks with the underlying optimization needs. We evaluate LoRA-RG on reasoning, mathematical, and code generation benchmarks, and show that it consistently achieves higher accuracy compared to fixed-rank LoRA and recent rank initialization and adaptation baselines under the same parameter budget.</abstract>
<identifier type="citekey">kwon-etal-2026-lora</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2000/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>45571</start>
<end>45587</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T LoRA-RG: Residual Gradient-Guided Rank Initialization and Expansion
%A Kwon, Dayoung
%A Lee, Jaeseong
%A Hwang, Seung-Won
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F kwon-etal-2026-lora
%X We study how to optimally allocate ranks in Low-Rank Adaptation (LoRA). Despite its importance, ranks are typically chosen manually and fixed across all modules. We argue that this practice is suboptimal, risking either under-parameterization, which hurts performance, or over-parameterization, which wastes memory and computation. To address this, we propose LoRA-RG, a gradient-guided framework that adjusts LoRA ranks through two stages: rank initialization and rank expansion. In the rank initialization stage, LoRA-RG estimates a minimal rank for each layer based on gradient information prior to training. During training, the rank expansion stage further increases ranks only when the current LoRA parameterization is insufficient to capture the remaining update demand. This design dynamically aligns LoRA ranks with the underlying optimization needs. We evaluate LoRA-RG on reasoning, mathematical, and code generation benchmarks, and show that it consistently achieves higher accuracy compared to fixed-rank LoRA and recent rank initialization and adaptation baselines under the same parameter budget.
%U https://aclanthology.org/2026.emnlp-main.2000/
%P 45571-45587
Markdown (Informal)
[LoRA-RG: Residual Gradient-Guided Rank Initialization and Expansion](https://aclanthology.org/2026.emnlp-main.2000/) (Kwon et al., EMNLP 2026)
ACL