@inproceedings{xu-etal-2026-small,
title = "Small Models, Fewer Heads, More Forgetting: Sparse Attention Fine-Tuning for {SLM} Continual Learning",
author = "Xu, Hao and
Song, Shezheng and
Peng, Long and
Liu, Xiaodong and
Li, Xiaopeng and
Jun, Ma and
Li, Shasha and
Yu, Jie",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1099/",
pages = "25205--25222",
ISBN = "979-8-89176-459-0",
abstract = "Small Language Models (SLMs) have been widely deployed in edge computing and mobile devices. Generally, these models need to be continuously updated to adapt to dynamic environments and evolving user needs. Therefore, the performance of SLMs in continual learning has become a critical research focus. Compared to LLMs with rich redundant representation spaces, SLMs are more likely to overwrite previous representations during new task learning due to their fewer attention heads and layers. To address this issue, we propose LightLoRA, a sparse fine-tuning method specifically designed for SLMs. This method constructs a conflict-aware importance mechanism by combining raw importance from SVD decomposition with task conflict degrees based on gradient direction consistency. By selecting and updating heads that provide high contributions and low conflicts, LightLoRA successfully maintains sufficient space for knowledge preservation. Experimental results show that the proposed method reaches state-of-the-art performance across multiple benchmarks without adding extra parameters. Code is released on \url{https://anonymous.4open.science/r/LightLoRA-077C}."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="xu-etal-2026-small">
<titleInfo>
<title>Small Models, Fewer Heads, More Forgetting: Sparse Attention Fine-Tuning for SLM Continual Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hao</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shezheng</namePart>
<namePart type="family">Song</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Long</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaodong</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaopeng</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ma</namePart>
<namePart type="family">Jun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shasha</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jie</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Small Language Models (SLMs) have been widely deployed in edge computing and mobile devices. Generally, these models need to be continuously updated to adapt to dynamic environments and evolving user needs. Therefore, the performance of SLMs in continual learning has become a critical research focus. Compared to LLMs with rich redundant representation spaces, SLMs are more likely to overwrite previous representations during new task learning due to their fewer attention heads and layers. To address this issue, we propose LightLoRA, a sparse fine-tuning method specifically designed for SLMs. This method constructs a conflict-aware importance mechanism by combining raw importance from SVD decomposition with task conflict degrees based on gradient direction consistency. By selecting and updating heads that provide high contributions and low conflicts, LightLoRA successfully maintains sufficient space for knowledge preservation. Experimental results show that the proposed method reaches state-of-the-art performance across multiple benchmarks without adding extra parameters. Code is released on https://anonymous.4open.science/r/LightLoRA-077C.</abstract>
<identifier type="citekey">xu-etal-2026-small</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1099/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>25205</start>
<end>25222</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Small Models, Fewer Heads, More Forgetting: Sparse Attention Fine-Tuning for SLM Continual Learning
%A Xu, Hao
%A Song, Shezheng
%A Peng, Long
%A Liu, Xiaodong
%A Li, Xiaopeng
%A Jun, Ma
%A Li, Shasha
%A Yu, Jie
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F xu-etal-2026-small
%X Small Language Models (SLMs) have been widely deployed in edge computing and mobile devices. Generally, these models need to be continuously updated to adapt to dynamic environments and evolving user needs. Therefore, the performance of SLMs in continual learning has become a critical research focus. Compared to LLMs with rich redundant representation spaces, SLMs are more likely to overwrite previous representations during new task learning due to their fewer attention heads and layers. To address this issue, we propose LightLoRA, a sparse fine-tuning method specifically designed for SLMs. This method constructs a conflict-aware importance mechanism by combining raw importance from SVD decomposition with task conflict degrees based on gradient direction consistency. By selecting and updating heads that provide high contributions and low conflicts, LightLoRA successfully maintains sufficient space for knowledge preservation. Experimental results show that the proposed method reaches state-of-the-art performance across multiple benchmarks without adding extra parameters. Code is released on https://anonymous.4open.science/r/LightLoRA-077C.
%U https://aclanthology.org/2026.emnlp-main.1099/
%P 25205-25222
Markdown (Informal)
[Small Models, Fewer Heads, More Forgetting: Sparse Attention Fine-Tuning for SLM Continual Learning](https://aclanthology.org/2026.emnlp-main.1099/) (Xu et al., EMNLP 2026)
ACL
- Hao Xu, Shezheng Song, Long Peng, Xiaodong Liu, Xiaopeng Li, Ma Jun, Shasha Li, and Jie Yu. 2026. Small Models, Fewer Heads, More Forgetting: Sparse Attention Fine-Tuning for SLM Continual Learning. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 25205–25222, Budapest, Hungary. Association for Computational Linguistics.