@inproceedings{lu-etal-2026-representation,
title = "On the Representation Geometry of {L}o{RA} Model Merging",
author = "Lu, Chenyang and
Li, Jiaru and
Zhao, Jinman and
Chen, Xinran and
Wang, Yining and
Cai, Renyi and
Li, Yuchen and
He, Chao",
editor = "Liakata, Maria and
Moreira, Viviane P. and
Zhang, Jiajun and
Jurgens, David",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2026",
month = jul,
year = "2026",
address = "San Diego, California, United States",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-acl.261/",
pages = "5289--5304",
ISBN = "979-8-89176-395-1",
abstract = "Low-Rank Adaptation (LoRA) is widely used for parameter-efficient fine-tuning, yet merging multiple task-specific LoRA updates without additional training remains challenging. Most existing LoRA merging methods rely on SVD-based alignment, which emphasizes globally shared structure across tasks. In this work, we show that LoRA merging performance can be further improved by combining SVD with CUR decomposition. Through a representation-level analysis, we find that SVD-based decompositions primarily model shared components across tasks, while CUR-based decompositions better preserve task-specific and localized updates. These two perspectives are geometrically misaligned and exhibit complementary advantages, revealing an inherent trade-off between capturing shared structure and preserving task-specific information in LoRA model merging. Guided by this analysis, we propose a training-free merging procedure that explicitly combines the shared structure captured by SVD with the task-specific components preserved by CUR. Experiments on both vision and language benchmarks demonstrate consistent improvements over existing gradient-free LoRA merging methods."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lu-etal-2026-representation">
<titleInfo>
<title>On the Representation Geometry of LoRA Model Merging</title>
</titleInfo>
<name type="personal">
<namePart type="given">Chenyang</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiaru</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jinman</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinran</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yining</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Renyi</namePart>
<namePart type="family">Cai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuchen</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chao</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: ACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Maria</namePart>
<namePart type="family">Liakata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Viviane</namePart>
<namePart type="given">P</namePart>
<namePart type="family">Moreira</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiajun</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Jurgens</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">San Diego, California, United States</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-395-1</identifier>
</relatedItem>
<abstract>Low-Rank Adaptation (LoRA) is widely used for parameter-efficient fine-tuning, yet merging multiple task-specific LoRA updates without additional training remains challenging. Most existing LoRA merging methods rely on SVD-based alignment, which emphasizes globally shared structure across tasks. In this work, we show that LoRA merging performance can be further improved by combining SVD with CUR decomposition. Through a representation-level analysis, we find that SVD-based decompositions primarily model shared components across tasks, while CUR-based decompositions better preserve task-specific and localized updates. These two perspectives are geometrically misaligned and exhibit complementary advantages, revealing an inherent trade-off between capturing shared structure and preserving task-specific information in LoRA model merging. Guided by this analysis, we propose a training-free merging procedure that explicitly combines the shared structure captured by SVD with the task-specific components preserved by CUR. Experiments on both vision and language benchmarks demonstrate consistent improvements over existing gradient-free LoRA merging methods.</abstract>
<identifier type="citekey">lu-etal-2026-representation</identifier>
<location>
<url>https://aclanthology.org/2026.findings-acl.261/</url>
</location>
<part>
<date>2026-07</date>
<extent unit="page">
<start>5289</start>
<end>5304</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T On the Representation Geometry of LoRA Model Merging
%A Lu, Chenyang
%A Li, Jiaru
%A Zhao, Jinman
%A Chen, Xinran
%A Wang, Yining
%A Cai, Renyi
%A Li, Yuchen
%A He, Chao
%Y Liakata, Maria
%Y Moreira, Viviane P.
%Y Zhang, Jiajun
%Y Jurgens, David
%S Findings of the Association for Computational Linguistics: ACL 2026
%D 2026
%8 July
%I Association for Computational Linguistics
%C San Diego, California, United States
%@ 979-8-89176-395-1
%F lu-etal-2026-representation
%X Low-Rank Adaptation (LoRA) is widely used for parameter-efficient fine-tuning, yet merging multiple task-specific LoRA updates without additional training remains challenging. Most existing LoRA merging methods rely on SVD-based alignment, which emphasizes globally shared structure across tasks. In this work, we show that LoRA merging performance can be further improved by combining SVD with CUR decomposition. Through a representation-level analysis, we find that SVD-based decompositions primarily model shared components across tasks, while CUR-based decompositions better preserve task-specific and localized updates. These two perspectives are geometrically misaligned and exhibit complementary advantages, revealing an inherent trade-off between capturing shared structure and preserving task-specific information in LoRA model merging. Guided by this analysis, we propose a training-free merging procedure that explicitly combines the shared structure captured by SVD with the task-specific components preserved by CUR. Experiments on both vision and language benchmarks demonstrate consistent improvements over existing gradient-free LoRA merging methods.
%U https://aclanthology.org/2026.findings-acl.261/
%P 5289-5304
Markdown (Informal)
[On the Representation Geometry of LoRA Model Merging](https://aclanthology.org/2026.findings-acl.261/) (Lu et al., Findings 2026)
ACL
- Chenyang Lu, Jiaru Li, Jinman Zhao, Xinran Chen, Yining Wang, Renyi Cai, Yuchen Li, and Chao He. 2026. On the Representation Geometry of LoRA Model Merging. In Findings of the Association for Computational Linguistics: ACL 2026, pages 5289–5304, San Diego, California, United States. Association for Computational Linguistics.