@article{liu-etal-2026-structtune,
title = "{S}truct{T}une: Structured Sparse Fine-Tuning for Efficient Adaptation of Large Language Models",
author = "Liu, Yuyang and
Po, Lai-Man and
Cheung, Kwok-Wai and
Jiang, Zeyu and
Wu, Haoxuan and
Li, Kun and
Xu, Xuyuan",
journal = "Transactions of the Association for Computational Linguistics",
volume = "14",
year = "2026",
address = "Cambridge, MA",
publisher = "MIT Press",
url = "https://aclanthology.org/2026.tacl-1.106/",
doi = "10.1162/tacl.a.803",
pages = "2318--2336",
abstract = "Parameter-efficient fine-tuning (PEFT) methods like LoRA have become crucial for adapting large language models under computational constraints. While LoRA focuses on dense weight updates, recent sparse methods propose updating only selected portions of pretrained weights. However, these sparse approaches predominantly use element-wise parameter selection, overlooking inherent structured patterns in LLM weight distributions. We introduce StructTune1, a PEFT method that uses entire rows or columns as basic units for identifying task-relevant sparse components. Our analysis reveals LLaMA models exhibit distinct structured patterns in pretrained weights, with fine-tuning updates showing structured amplification rather than uniform changes. We develop a sensitivity-based framework to efficiently identify task-relevant weight structures without full fine-tuning. Experiments on mathematical reasoning and code generation demonstrate StructTune{'}s structured sparse updates outperform both dense LoRA and element-wise sparse methods while maintaining computational efficiency. Ablation studies validate our sensitivity-based selection strategy, confirming the efficacy of exploiting structured weight patterns in PEFT."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-structtune">
<titleInfo>
<title>StructTune: Structured Sparse Fine-Tuning for Efficient Adaptation of Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yuyang</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lai-Man</namePart>
<namePart type="family">Po</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kwok-Wai</namePart>
<namePart type="family">Cheung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zeyu</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haoxuan</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kun</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xuyuan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<genre authority="bibutilsgt">journal article</genre>
<relatedItem type="host">
<titleInfo>
<title>Transactions of the Association for Computational Linguistics</title>
</titleInfo>
<originInfo>
<issuance>continuing</issuance>
<publisher>MIT Press</publisher>
<place>
<placeTerm type="text">Cambridge, MA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">periodical</genre>
<genre authority="bibutilsgt">academic journal</genre>
</relatedItem>
<abstract>Parameter-efficient fine-tuning (PEFT) methods like LoRA have become crucial for adapting large language models under computational constraints. While LoRA focuses on dense weight updates, recent sparse methods propose updating only selected portions of pretrained weights. However, these sparse approaches predominantly use element-wise parameter selection, overlooking inherent structured patterns in LLM weight distributions. We introduce StructTune1, a PEFT method that uses entire rows or columns as basic units for identifying task-relevant sparse components. Our analysis reveals LLaMA models exhibit distinct structured patterns in pretrained weights, with fine-tuning updates showing structured amplification rather than uniform changes. We develop a sensitivity-based framework to efficiently identify task-relevant weight structures without full fine-tuning. Experiments on mathematical reasoning and code generation demonstrate StructTune’s structured sparse updates outperform both dense LoRA and element-wise sparse methods while maintaining computational efficiency. Ablation studies validate our sensitivity-based selection strategy, confirming the efficacy of exploiting structured weight patterns in PEFT.</abstract>
<identifier type="citekey">liu-etal-2026-structtune</identifier>
<identifier type="doi">10.1162/tacl.a.803</identifier>
<location>
<url>https://aclanthology.org/2026.tacl-1.106/</url>
</location>
<part>
<date>2026</date>
<detail type="volume"><number>14</number></detail>
<extent unit="page">
<start>2318</start>
<end>2336</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Journal Article
%T StructTune: Structured Sparse Fine-Tuning for Efficient Adaptation of Large Language Models
%A Liu, Yuyang
%A Po, Lai-Man
%A Cheung, Kwok-Wai
%A Jiang, Zeyu
%A Wu, Haoxuan
%A Li, Kun
%A Xu, Xuyuan
%J Transactions of the Association for Computational Linguistics
%D 2026
%V 14
%I MIT Press
%C Cambridge, MA
%F liu-etal-2026-structtune
%X Parameter-efficient fine-tuning (PEFT) methods like LoRA have become crucial for adapting large language models under computational constraints. While LoRA focuses on dense weight updates, recent sparse methods propose updating only selected portions of pretrained weights. However, these sparse approaches predominantly use element-wise parameter selection, overlooking inherent structured patterns in LLM weight distributions. We introduce StructTune1, a PEFT method that uses entire rows or columns as basic units for identifying task-relevant sparse components. Our analysis reveals LLaMA models exhibit distinct structured patterns in pretrained weights, with fine-tuning updates showing structured amplification rather than uniform changes. We develop a sensitivity-based framework to efficiently identify task-relevant weight structures without full fine-tuning. Experiments on mathematical reasoning and code generation demonstrate StructTune’s structured sparse updates outperform both dense LoRA and element-wise sparse methods while maintaining computational efficiency. Ablation studies validate our sensitivity-based selection strategy, confirming the efficacy of exploiting structured weight patterns in PEFT.
%R 10.1162/tacl.a.803
%U https://aclanthology.org/2026.tacl-1.106/
%U https://doi.org/10.1162/tacl.a.803
%P 2318-2336
Markdown (Informal)
[StructTune: Structured Sparse Fine-Tuning for Efficient Adaptation of Large Language Models](https://aclanthology.org/2026.tacl-1.106/) (Liu et al., TACL 2026)
ACL