@inproceedings{huang-etal-2026-peft,
title = "{PEFT}-Arena: Understanding Parameter-Efficient Finetuning from a Stability-Plasticity Perspective",
author = {Huang, Yangyi and
Peng, Ruotian and
Qiu, Zeju and
Kang, Jiale and
Wen, Yandong and
Sch{\"o}lkopf, Bernhard and
Liu, Weiyang},
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1930/",
pages = "41146--41172",
ISBN = "979-8-89176-462-0",
abstract = "Parameter-efficient finetuning (PEFT) has become the standard approach for adapting large language models, yet evaluations largely emphasize downstream accuracy while overlooking the retention of pretrained capabilities. We argue that PEFT should be assessed through the stability-plasticity dilemma: the trade-off between target-task adaptation and resistance to forgetting. We introduce PEFT-Arena, a benchmark that jointly measures downstream performance and general capability retention. Across the studied settings, methods exhibit distinct stability-plasticity profiles, and orthogonal finetuning often lies on a favorable frontier under comparable parameter budgets. We analyze the internal changes associated with these external trade-offs from two complementary geometric perspectives. In weight space, spectral profiles characterize how parameterizations interact with the pretrained singular-value structure. In activation space, representation-similarity diagnostics show a strong association between forgetting and non-isometric distortion on general-evaluation data. Together, these views suggest that preserving pretrained spectral and representational geometry is an important inductive bias for retention, with OFT{'}s spectrum-preserving updates providing a particularly clear case. Finally, parameterization-aware interpolation shows that final SFT checkpoints can lie beyond a better target-retention operating point. We use post-hoc rewinding as a case study of controlling this trade-off."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="huang-etal-2026-peft">
<titleInfo>
<title>PEFT-Arena: Understanding Parameter-Efficient Finetuning from a Stability-Plasticity Perspective</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yangyi</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruotian</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zeju</namePart>
<namePart type="family">Qiu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiale</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yandong</namePart>
<namePart type="family">Wen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bernhard</namePart>
<namePart type="family">Schölkopf</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weiyang</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Parameter-efficient finetuning (PEFT) has become the standard approach for adapting large language models, yet evaluations largely emphasize downstream accuracy while overlooking the retention of pretrained capabilities. We argue that PEFT should be assessed through the stability-plasticity dilemma: the trade-off between target-task adaptation and resistance to forgetting. We introduce PEFT-Arena, a benchmark that jointly measures downstream performance and general capability retention. Across the studied settings, methods exhibit distinct stability-plasticity profiles, and orthogonal finetuning often lies on a favorable frontier under comparable parameter budgets. We analyze the internal changes associated with these external trade-offs from two complementary geometric perspectives. In weight space, spectral profiles characterize how parameterizations interact with the pretrained singular-value structure. In activation space, representation-similarity diagnostics show a strong association between forgetting and non-isometric distortion on general-evaluation data. Together, these views suggest that preserving pretrained spectral and representational geometry is an important inductive bias for retention, with OFT’s spectrum-preserving updates providing a particularly clear case. Finally, parameterization-aware interpolation shows that final SFT checkpoints can lie beyond a better target-retention operating point. We use post-hoc rewinding as a case study of controlling this trade-off.</abstract>
<identifier type="citekey">huang-etal-2026-peft</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1930/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>41146</start>
<end>41172</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T PEFT-Arena: Understanding Parameter-Efficient Finetuning from a Stability-Plasticity Perspective
%A Huang, Yangyi
%A Peng, Ruotian
%A Qiu, Zeju
%A Kang, Jiale
%A Wen, Yandong
%A Schölkopf, Bernhard
%A Liu, Weiyang
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F huang-etal-2026-peft
%X Parameter-efficient finetuning (PEFT) has become the standard approach for adapting large language models, yet evaluations largely emphasize downstream accuracy while overlooking the retention of pretrained capabilities. We argue that PEFT should be assessed through the stability-plasticity dilemma: the trade-off between target-task adaptation and resistance to forgetting. We introduce PEFT-Arena, a benchmark that jointly measures downstream performance and general capability retention. Across the studied settings, methods exhibit distinct stability-plasticity profiles, and orthogonal finetuning often lies on a favorable frontier under comparable parameter budgets. We analyze the internal changes associated with these external trade-offs from two complementary geometric perspectives. In weight space, spectral profiles characterize how parameterizations interact with the pretrained singular-value structure. In activation space, representation-similarity diagnostics show a strong association between forgetting and non-isometric distortion on general-evaluation data. Together, these views suggest that preserving pretrained spectral and representational geometry is an important inductive bias for retention, with OFT’s spectrum-preserving updates providing a particularly clear case. Finally, parameterization-aware interpolation shows that final SFT checkpoints can lie beyond a better target-retention operating point. We use post-hoc rewinding as a case study of controlling this trade-off.
%U https://aclanthology.org/2026.findings-emnlp.1930/
%P 41146-41172
Markdown (Informal)
[PEFT-Arena: Understanding Parameter-Efficient Finetuning from a Stability-Plasticity Perspective](https://aclanthology.org/2026.findings-emnlp.1930/) (Huang et al., Findings 2026)
ACL