@inproceedings{bhattacharyya-etal-2026-forget,
title = "Forget-to-Focus: Targeted Unlearning for Domain Specialization in {LLM}s",
author = "Bhattacharyya, Chaitali and
Jang, Shinhyoung and
Kim, Seohyun and
Lee, Junyoung and
Kim, Yeseong",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.952/",
pages = "20059--20085",
ISBN = "979-8-89176-462-0",
abstract = "Supervised fine-tuning is the standard approach for adapting pretrained large language models to specialized domains, yet it treats the pretrained checkpoint as a fixed and neutral initialization. In practice, pretrained models encode broad general-domain priors that can conflict with target-domain learning. We propose Forget-to-Focus (F2F), a two-stage domain-specialization framework that first applies controlled unlearning to reshape the pretrained checkpoint and then performs ordinary supervised fine-tuning from the resulting initialization. A key challenge is deciding what to forget. We introduce NG-DS, a neuron-guided data selection method that identifies pretraining-dominant and domain-dominant feed-forward neurons, then selects forget examples that strongly activate the former while weakly activating the latter. This model-aware forget set directs unlearning toward pretraining-like behavior while reducing the risk of erasing target-domain-relevant knowledge.We evaluate F2F on five LLM backbones spanning 0.6B-72B parameters across coding, medical question answering, and mathematical reasoning. F2F consistently improves over standard SFT and adaptation baselines. Across coding, medical QA, and mathematical reasoning, F2F consistently improves over standard tuning. On HumanEval, it improves over SFT by +13.84 for Qwen3-0.6B, +7.86 for LLaMA3.1-8B, and +10.14 for LLaMA2-13B. On PubMedQA, it improves Qwen3-0.6B from 62.60 to 80.50, showing that targeted preparatory forgetting provides a better initialization for tuning."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="bhattacharyya-etal-2026-forget">
<titleInfo>
<title>Forget-to-Focus: Targeted Unlearning for Domain Specialization in LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Chaitali</namePart>
<namePart type="family">Bhattacharyya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shinhyoung</namePart>
<namePart type="family">Jang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Seohyun</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junyoung</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yeseong</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Supervised fine-tuning is the standard approach for adapting pretrained large language models to specialized domains, yet it treats the pretrained checkpoint as a fixed and neutral initialization. In practice, pretrained models encode broad general-domain priors that can conflict with target-domain learning. We propose Forget-to-Focus (F2F), a two-stage domain-specialization framework that first applies controlled unlearning to reshape the pretrained checkpoint and then performs ordinary supervised fine-tuning from the resulting initialization. A key challenge is deciding what to forget. We introduce NG-DS, a neuron-guided data selection method that identifies pretraining-dominant and domain-dominant feed-forward neurons, then selects forget examples that strongly activate the former while weakly activating the latter. This model-aware forget set directs unlearning toward pretraining-like behavior while reducing the risk of erasing target-domain-relevant knowledge.We evaluate F2F on five LLM backbones spanning 0.6B-72B parameters across coding, medical question answering, and mathematical reasoning. F2F consistently improves over standard SFT and adaptation baselines. Across coding, medical QA, and mathematical reasoning, F2F consistently improves over standard tuning. On HumanEval, it improves over SFT by +13.84 for Qwen3-0.6B, +7.86 for LLaMA3.1-8B, and +10.14 for LLaMA2-13B. On PubMedQA, it improves Qwen3-0.6B from 62.60 to 80.50, showing that targeted preparatory forgetting provides a better initialization for tuning.</abstract>
<identifier type="citekey">bhattacharyya-etal-2026-forget</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.952/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>20059</start>
<end>20085</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Forget-to-Focus: Targeted Unlearning for Domain Specialization in LLMs
%A Bhattacharyya, Chaitali
%A Jang, Shinhyoung
%A Kim, Seohyun
%A Lee, Junyoung
%A Kim, Yeseong
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F bhattacharyya-etal-2026-forget
%X Supervised fine-tuning is the standard approach for adapting pretrained large language models to specialized domains, yet it treats the pretrained checkpoint as a fixed and neutral initialization. In practice, pretrained models encode broad general-domain priors that can conflict with target-domain learning. We propose Forget-to-Focus (F2F), a two-stage domain-specialization framework that first applies controlled unlearning to reshape the pretrained checkpoint and then performs ordinary supervised fine-tuning from the resulting initialization. A key challenge is deciding what to forget. We introduce NG-DS, a neuron-guided data selection method that identifies pretraining-dominant and domain-dominant feed-forward neurons, then selects forget examples that strongly activate the former while weakly activating the latter. This model-aware forget set directs unlearning toward pretraining-like behavior while reducing the risk of erasing target-domain-relevant knowledge.We evaluate F2F on five LLM backbones spanning 0.6B-72B parameters across coding, medical question answering, and mathematical reasoning. F2F consistently improves over standard SFT and adaptation baselines. Across coding, medical QA, and mathematical reasoning, F2F consistently improves over standard tuning. On HumanEval, it improves over SFT by +13.84 for Qwen3-0.6B, +7.86 for LLaMA3.1-8B, and +10.14 for LLaMA2-13B. On PubMedQA, it improves Qwen3-0.6B from 62.60 to 80.50, showing that targeted preparatory forgetting provides a better initialization for tuning.
%U https://aclanthology.org/2026.findings-emnlp.952/
%P 20059-20085
Markdown (Informal)
[Forget-to-Focus: Targeted Unlearning for Domain Specialization in LLMs](https://aclanthology.org/2026.findings-emnlp.952/) (Bhattacharyya et al., Findings 2026)
ACL