@inproceedings{wang-2025-optimizing,
title = "Optimizing Lifelong Fine-Tuning for Multiple Tasks via Dataless Distribution Replay",
author = "Wang, Zhenxing",
editor = "Rambow, Owen and
Wanner, Leo and
Apidianaki, Marianna and
Al-Khalifa, Hend and
Eugenio, Barbara Di and
Schockaert, Steven",
booktitle = "Proceedings of the 31st International Conference on Computational Linguistics",
month = jan,
year = "2025",
address = "Abu Dhabi, UAE",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2025.coling-main.746/",
pages = "11261--11273",
abstract = "The recent emergence of various large language models, which can be fine-tuned with minimal instruction data, has demonstrated impressive performance across various tasks. However, a phenomenon of forgetting occurs during life- long fine-tuning because training on new tasks interferes with the previously acquired knowl- edge. To mitigate catastrophic forgetting, con- ventional data replay methods achieve high per- formance, but at the cost of compromising data privacy and security. This paper introduces a dataless distribution replay approach for life- long fine-tuning. Concretely, the distribution distillation is applied to replay the output dis- tribution of the linear layers at previous task stages. The optimal solution for this distri- bution replay can be directly computed using the retained inner product matrix of the input data, thereby eliminating the need for previ- ous data. Additionally, Singular Value Decom- position (SVD) and module accumulation are employed to further enhance the performance of dataless distribution replay method. Finally, the evaluation is conducted in a lifelong fine- tuning scenario involving multiple tasks. The experimental results and analysis show that the proposed method achieves significant improve- ments compared to several strong lifelong fine- tuning methods."
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-2025-optimizing">
<titleInfo>
<title>Optimizing Lifelong Fine-Tuning for Multiple Tasks via Dataless Distribution Replay</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zhenxing</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2025-01</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 31st International Conference on Computational Linguistics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Owen</namePart>
<namePart type="family">Rambow</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leo</namePart>
<namePart type="family">Wanner</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marianna</namePart>
<namePart type="family">Apidianaki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hend</namePart>
<namePart type="family">Al-Khalifa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="given">Di</namePart>
<namePart type="family">Eugenio</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Steven</namePart>
<namePart type="family">Schockaert</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Abu Dhabi, UAE</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>The recent emergence of various large language models, which can be fine-tuned with minimal instruction data, has demonstrated impressive performance across various tasks. However, a phenomenon of forgetting occurs during life- long fine-tuning because training on new tasks interferes with the previously acquired knowl- edge. To mitigate catastrophic forgetting, con- ventional data replay methods achieve high per- formance, but at the cost of compromising data privacy and security. This paper introduces a dataless distribution replay approach for life- long fine-tuning. Concretely, the distribution distillation is applied to replay the output dis- tribution of the linear layers at previous task stages. The optimal solution for this distri- bution replay can be directly computed using the retained inner product matrix of the input data, thereby eliminating the need for previ- ous data. Additionally, Singular Value Decom- position (SVD) and module accumulation are employed to further enhance the performance of dataless distribution replay method. Finally, the evaluation is conducted in a lifelong fine- tuning scenario involving multiple tasks. The experimental results and analysis show that the proposed method achieves significant improve- ments compared to several strong lifelong fine- tuning methods.</abstract>
<identifier type="citekey">wang-2025-optimizing</identifier>
<location>
<url>https://aclanthology.org/2025.coling-main.746/</url>
</location>
<part>
<date>2025-01</date>
<extent unit="page">
<start>11261</start>
<end>11273</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Optimizing Lifelong Fine-Tuning for Multiple Tasks via Dataless Distribution Replay
%A Wang, Zhenxing
%Y Rambow, Owen
%Y Wanner, Leo
%Y Apidianaki, Marianna
%Y Al-Khalifa, Hend
%Y Eugenio, Barbara Di
%Y Schockaert, Steven
%S Proceedings of the 31st International Conference on Computational Linguistics
%D 2025
%8 January
%I Association for Computational Linguistics
%C Abu Dhabi, UAE
%F wang-2025-optimizing
%X The recent emergence of various large language models, which can be fine-tuned with minimal instruction data, has demonstrated impressive performance across various tasks. However, a phenomenon of forgetting occurs during life- long fine-tuning because training on new tasks interferes with the previously acquired knowl- edge. To mitigate catastrophic forgetting, con- ventional data replay methods achieve high per- formance, but at the cost of compromising data privacy and security. This paper introduces a dataless distribution replay approach for life- long fine-tuning. Concretely, the distribution distillation is applied to replay the output dis- tribution of the linear layers at previous task stages. The optimal solution for this distri- bution replay can be directly computed using the retained inner product matrix of the input data, thereby eliminating the need for previ- ous data. Additionally, Singular Value Decom- position (SVD) and module accumulation are employed to further enhance the performance of dataless distribution replay method. Finally, the evaluation is conducted in a lifelong fine- tuning scenario involving multiple tasks. The experimental results and analysis show that the proposed method achieves significant improve- ments compared to several strong lifelong fine- tuning methods.
%U https://aclanthology.org/2025.coling-main.746/
%P 11261-11273
Markdown (Informal)
[Optimizing Lifelong Fine-Tuning for Multiple Tasks via Dataless Distribution Replay](https://aclanthology.org/2025.coling-main.746/) (Wang, COLING 2025)
ACL