@inproceedings{moskovskiy-panchenko-2026-k,
title = "K-{FORGE}: Fisher-Guided Initialization for Preference-Based {LLM} Unlearning",
author = "Moskovskiy, Daniil and
Panchenko, Alexander",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.214/",
pages = "4233--4248",
ISBN = "979-8-89176-444-6",
abstract = "Preference-based LLM unlearning methods such as NPO and SimNPO must discover directions that suppress forget-set behavior without damaging retained capability. K-FORGE uses forget- and retain-set Fisher information to construct a retain-aware low-rank edit, then starts NPO or SimNPO from the edited checkpoint. On the TOFU forget10 split, where 10{\%} of the original dataset is designated for forgetting, stronger one-shot edits lower both Forget Q/A Probability and Model Utility. With Llama 3.2 1B, K-FORGE reduces Forget Q/A Probability by 22.3{\%} under NPO while improving utility and using less total time and compute than scratch. SimNPO shows a smaller descriptive probability reduction with lower utility. On the same split, random and weight-SVD initializers yield higher forget probability than K-FORGE at 50 and 100 steps. Experiments with Gemma 3 1B also yield lower mean forget probability for both optimizers at 100 steps. MUSE results vary by metric, and direct relearning recovers forgotten behavior."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="moskovskiy-panchenko-2026-k">
<titleInfo>
<title>K-FORGE: Fisher-Guided Initialization for Preference-Based LLM Unlearning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Daniil</namePart>
<namePart type="family">Moskovskiy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexander</namePart>
<namePart type="family">Panchenko</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>Preference-based LLM unlearning methods such as NPO and SimNPO must discover directions that suppress forget-set behavior without damaging retained capability. K-FORGE uses forget- and retain-set Fisher information to construct a retain-aware low-rank edit, then starts NPO or SimNPO from the edited checkpoint. On the TOFU forget10 split, where 10% of the original dataset is designated for forgetting, stronger one-shot edits lower both Forget Q/A Probability and Model Utility. With Llama 3.2 1B, K-FORGE reduces Forget Q/A Probability by 22.3% under NPO while improving utility and using less total time and compute than scratch. SimNPO shows a smaller descriptive probability reduction with lower utility. On the same split, random and weight-SVD initializers yield higher forget probability than K-FORGE at 50 and 100 steps. Experiments with Gemma 3 1B also yield lower mean forget probability for both optimizers at 100 steps. MUSE results vary by metric, and direct relearning recovers forgotten behavior.</abstract>
<identifier type="citekey">moskovskiy-panchenko-2026-k</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.214/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>4233</start>
<end>4248</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T K-FORGE: Fisher-Guided Initialization for Preference-Based LLM Unlearning
%A Moskovskiy, Daniil
%A Panchenko, Alexander
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F moskovskiy-panchenko-2026-k
%X Preference-based LLM unlearning methods such as NPO and SimNPO must discover directions that suppress forget-set behavior without damaging retained capability. K-FORGE uses forget- and retain-set Fisher information to construct a retain-aware low-rank edit, then starts NPO or SimNPO from the edited checkpoint. On the TOFU forget10 split, where 10% of the original dataset is designated for forgetting, stronger one-shot edits lower both Forget Q/A Probability and Model Utility. With Llama 3.2 1B, K-FORGE reduces Forget Q/A Probability by 22.3% under NPO while improving utility and using less total time and compute than scratch. SimNPO shows a smaller descriptive probability reduction with lower utility. On the same split, random and weight-SVD initializers yield higher forget probability than K-FORGE at 50 and 100 steps. Experiments with Gemma 3 1B also yield lower mean forget probability for both optimizers at 100 steps. MUSE results vary by metric, and direct relearning recovers forgotten behavior.
%U https://aclanthology.org/2026.findings-aacl.214/
%P 4233-4248
Markdown (Informal)
[K-FORGE: Fisher-Guided Initialization for Preference-Based LLM Unlearning](https://aclanthology.org/2026.findings-aacl.214/) (Moskovskiy & Panchenko, Findings 2026)
ACL