@inproceedings{han-etal-2026-ebs,
title = "{EBS}: Efficient Block-Sparse Framework for Continual Learning of Large Language Models",
author = "Han, Zhixin and
Gao, Hang and
Bai, Yinhao and
Zhao, Yuhua and
Luo, Bitong and
li, Zhiyu and
Hu, Mengting and
Huang, Minlie",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2462/",
pages = "52983--52993",
ISBN = "979-8-89176-462-0",
abstract = "By continuously acquiring new knowledge, continual learning enables large language models (LLMs) to remain adaptive, accurate, and relevant as the world evolves. In continual learning, tasks arrive one after another and the model learns them sequentially. As more tasks are introduced, existing methods for learning a new task impose increasing computational demands. However, we find that the computational overhead, which gradually increases as the number of tasks grows, can be mitigated. To limit the continual increase in computational costs, we introduce a simple but efficient block-sparse (EBS) framework for continual learning of LLMs. In EBS, we employ a dedicated, task-specific low-rank adaptation (LoRA) block for each task. When training on a new task, only one LoRA block is loaded into GPU memory and involved in computation, while at inference each input activates at most two LoRA blocks. This design ensures that the computational cost remains independent of the number of tasks, allowing EBS to maintain high efficiency even when the sequence of tasks becomes very large. We offer a simple implementation that works across different model sizes and transformer architectures (decoder-only and encoder-decoder) while consistently delivering both high efficiency and strong task performance. Experimental results on two continual learning benchmarks demonstrate that EBS achieves state-of-the-art performance on the T5 and LLaMA 2 models."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="han-etal-2026-ebs">
<titleInfo>
<title>EBS: Efficient Block-Sparse Framework for Continual Learning of Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zhixin</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hang</namePart>
<namePart type="family">Gao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yinhao</namePart>
<namePart type="family">Bai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuhua</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bitong</namePart>
<namePart type="family">Luo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhiyu</namePart>
<namePart type="family">li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mengting</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minlie</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>By continuously acquiring new knowledge, continual learning enables large language models (LLMs) to remain adaptive, accurate, and relevant as the world evolves. In continual learning, tasks arrive one after another and the model learns them sequentially. As more tasks are introduced, existing methods for learning a new task impose increasing computational demands. However, we find that the computational overhead, which gradually increases as the number of tasks grows, can be mitigated. To limit the continual increase in computational costs, we introduce a simple but efficient block-sparse (EBS) framework for continual learning of LLMs. In EBS, we employ a dedicated, task-specific low-rank adaptation (LoRA) block for each task. When training on a new task, only one LoRA block is loaded into GPU memory and involved in computation, while at inference each input activates at most two LoRA blocks. This design ensures that the computational cost remains independent of the number of tasks, allowing EBS to maintain high efficiency even when the sequence of tasks becomes very large. We offer a simple implementation that works across different model sizes and transformer architectures (decoder-only and encoder-decoder) while consistently delivering both high efficiency and strong task performance. Experimental results on two continual learning benchmarks demonstrate that EBS achieves state-of-the-art performance on the T5 and LLaMA 2 models.</abstract>
<identifier type="citekey">han-etal-2026-ebs</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2462/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>52983</start>
<end>52993</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T EBS: Efficient Block-Sparse Framework for Continual Learning of Large Language Models
%A Han, Zhixin
%A Gao, Hang
%A Bai, Yinhao
%A Zhao, Yuhua
%A Luo, Bitong
%A li, Zhiyu
%A Hu, Mengting
%A Huang, Minlie
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F han-etal-2026-ebs
%X By continuously acquiring new knowledge, continual learning enables large language models (LLMs) to remain adaptive, accurate, and relevant as the world evolves. In continual learning, tasks arrive one after another and the model learns them sequentially. As more tasks are introduced, existing methods for learning a new task impose increasing computational demands. However, we find that the computational overhead, which gradually increases as the number of tasks grows, can be mitigated. To limit the continual increase in computational costs, we introduce a simple but efficient block-sparse (EBS) framework for continual learning of LLMs. In EBS, we employ a dedicated, task-specific low-rank adaptation (LoRA) block for each task. When training on a new task, only one LoRA block is loaded into GPU memory and involved in computation, while at inference each input activates at most two LoRA blocks. This design ensures that the computational cost remains independent of the number of tasks, allowing EBS to maintain high efficiency even when the sequence of tasks becomes very large. We offer a simple implementation that works across different model sizes and transformer architectures (decoder-only and encoder-decoder) while consistently delivering both high efficiency and strong task performance. Experimental results on two continual learning benchmarks demonstrate that EBS achieves state-of-the-art performance on the T5 and LLaMA 2 models.
%U https://aclanthology.org/2026.findings-emnlp.2462/
%P 52983-52993
Markdown (Informal)
[EBS: Efficient Block-Sparse Framework for Continual Learning of Large Language Models](https://aclanthology.org/2026.findings-emnlp.2462/) (Han et al., Findings 2026)
ACL
- Zhixin Han, Hang Gao, Yinhao Bai, Yuhua Zhao, Bitong Luo, Zhiyu li, Mengting Hu, and Minlie Huang. 2026. EBS: Efficient Block-Sparse Framework for Continual Learning of Large Language Models. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 52983–52993, Budapest, Hungary. Association for Computational Linguistics.