@inproceedings{tang-etal-2026-finetune,
title = "Finetune Once: Decoupling General {\&} Domain Learning with Dynamic Anchor Annealing",
author = "Tang, Yang and
Liu, Ruijie and
Wang, Yifan and
Li, Shiyu and
Li, Peiming and
Wei, Zheng",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2532/",
pages = "57590--57607",
ISBN = "979-8-89176-459-0",
abstract = "Fine-tuning large language models on domain data is a standard practice, yet vanilla pipelines rely on a costly data mixture sweep to avoid catastrophic forgetting of general capabilities. We propose Dynamic Anchor Annealing (DAA), a full-parameter fine-tuning framework that decouples general and domain learning at the gradient level rather than at the data level. DAA estimates a global gradient anchor once on general data through zero learning rate training, anchors the domain gradient toward this anchor during fine-tuning, modulates step sizes by the cosine similarity between the two gradients, and combines this with annealing learning. Under this design, fine-tuning relies solely on domain data and does not lose general capability. Across four domains and three base models, DAA improves the joint domain and general harmonic mean by 12.5 points on average over direct fine-tuning and by 3.8 points over vanilla data mixture fine-tuning, and the margin widens to 66.9 points on a temporal out-of-distribution setting where direct fine-tuning collapses. Our headline 91{\%} reduction in GPU hours is a workflow level saving from removing the mixture ratio sweep rather than a per run speedup, since the per run cost of DAA is comparable to direct fine-tuning."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="tang-etal-2026-finetune">
<titleInfo>
<title>Finetune Once: Decoupling General & Domain Learning with Dynamic Anchor Annealing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yang</namePart>
<namePart type="family">Tang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruijie</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yifan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shiyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peiming</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zheng</namePart>
<namePart type="family">Wei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Fine-tuning large language models on domain data is a standard practice, yet vanilla pipelines rely on a costly data mixture sweep to avoid catastrophic forgetting of general capabilities. We propose Dynamic Anchor Annealing (DAA), a full-parameter fine-tuning framework that decouples general and domain learning at the gradient level rather than at the data level. DAA estimates a global gradient anchor once on general data through zero learning rate training, anchors the domain gradient toward this anchor during fine-tuning, modulates step sizes by the cosine similarity between the two gradients, and combines this with annealing learning. Under this design, fine-tuning relies solely on domain data and does not lose general capability. Across four domains and three base models, DAA improves the joint domain and general harmonic mean by 12.5 points on average over direct fine-tuning and by 3.8 points over vanilla data mixture fine-tuning, and the margin widens to 66.9 points on a temporal out-of-distribution setting where direct fine-tuning collapses. Our headline 91% reduction in GPU hours is a workflow level saving from removing the mixture ratio sweep rather than a per run speedup, since the per run cost of DAA is comparable to direct fine-tuning.</abstract>
<identifier type="citekey">tang-etal-2026-finetune</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2532/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>57590</start>
<end>57607</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Finetune Once: Decoupling General & Domain Learning with Dynamic Anchor Annealing
%A Tang, Yang
%A Liu, Ruijie
%A Wang, Yifan
%A Li, Shiyu
%A Li, Peiming
%A Wei, Zheng
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F tang-etal-2026-finetune
%X Fine-tuning large language models on domain data is a standard practice, yet vanilla pipelines rely on a costly data mixture sweep to avoid catastrophic forgetting of general capabilities. We propose Dynamic Anchor Annealing (DAA), a full-parameter fine-tuning framework that decouples general and domain learning at the gradient level rather than at the data level. DAA estimates a global gradient anchor once on general data through zero learning rate training, anchors the domain gradient toward this anchor during fine-tuning, modulates step sizes by the cosine similarity between the two gradients, and combines this with annealing learning. Under this design, fine-tuning relies solely on domain data and does not lose general capability. Across four domains and three base models, DAA improves the joint domain and general harmonic mean by 12.5 points on average over direct fine-tuning and by 3.8 points over vanilla data mixture fine-tuning, and the margin widens to 66.9 points on a temporal out-of-distribution setting where direct fine-tuning collapses. Our headline 91% reduction in GPU hours is a workflow level saving from removing the mixture ratio sweep rather than a per run speedup, since the per run cost of DAA is comparable to direct fine-tuning.
%U https://aclanthology.org/2026.emnlp-main.2532/
%P 57590-57607
Markdown (Informal)
[Finetune Once: Decoupling General & Domain Learning with Dynamic Anchor Annealing](https://aclanthology.org/2026.emnlp-main.2532/) (Tang et al., EMNLP 2026)
ACL