@inproceedings{lin-etal-2026-beyond,
title = "Beyond Task Boundaries: Instruction-Level Parameter Isolation for Supervised Fine-Tuning",
author = "Lin, Zekai and
Xiang, Long and
Liang, Di and
Liu, Peiyang and
Hu, Zhanghao and
Hong, Mengze and
Wu, Xianjie and
Peng, Minlong",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.287/",
pages = "6571--6595",
ISBN = "979-8-89176-459-0",
abstract = "Modern supervised fine-tuning (SFT) trains large language models on heterogeneous instruction mixtures, where the effective unit of optimization is an individual instruction-response pair rather than a coarse task label. Existing parameter isolation methods mitigate interference by assigning protected or trainable subspaces at the task or training stage level, but these units are often misaligned with actual parameter usage: instructions from the same task may require conflicting computations, while instructions from different tasks may share latent capabilities. We propose \textbf{Instruction-Level Parameter Isolation (IPI)}, a fine-tuning framework that routes each instruction{'}s update to a sparse, capability-conditioned parameter subspace. IPI builds a multi-view instruction representation, predicts importance factors with a lightweight router, and uses an identity-trick reparameterization that preserves dense forward computation while exposing sparse base-model gradients and a task-loss saliency signal for router learning. Across heterogeneous instruction tuning benchmarks and multiple backbone models, IPI consistently improves performance over strong fine-tuning and parameter-isolation baselines. Further analyses show that instruction-conditioned masks reduce destructive gradient interactions and capture optimization structure that task labels miss."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lin-etal-2026-beyond">
<titleInfo>
<title>Beyond Task Boundaries: Instruction-Level Parameter Isolation for Supervised Fine-Tuning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zekai</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Long</namePart>
<namePart type="family">Xiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peiyang</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhanghao</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mengze</namePart>
<namePart type="family">Hong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xianjie</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minlong</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Modern supervised fine-tuning (SFT) trains large language models on heterogeneous instruction mixtures, where the effective unit of optimization is an individual instruction-response pair rather than a coarse task label. Existing parameter isolation methods mitigate interference by assigning protected or trainable subspaces at the task or training stage level, but these units are often misaligned with actual parameter usage: instructions from the same task may require conflicting computations, while instructions from different tasks may share latent capabilities. We propose Instruction-Level Parameter Isolation (IPI), a fine-tuning framework that routes each instruction’s update to a sparse, capability-conditioned parameter subspace. IPI builds a multi-view instruction representation, predicts importance factors with a lightweight router, and uses an identity-trick reparameterization that preserves dense forward computation while exposing sparse base-model gradients and a task-loss saliency signal for router learning. Across heterogeneous instruction tuning benchmarks and multiple backbone models, IPI consistently improves performance over strong fine-tuning and parameter-isolation baselines. Further analyses show that instruction-conditioned masks reduce destructive gradient interactions and capture optimization structure that task labels miss.</abstract>
<identifier type="citekey">lin-etal-2026-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.287/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>6571</start>
<end>6595</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Task Boundaries: Instruction-Level Parameter Isolation for Supervised Fine-Tuning
%A Lin, Zekai
%A Xiang, Long
%A Liang, Di
%A Liu, Peiyang
%A Hu, Zhanghao
%A Hong, Mengze
%A Wu, Xianjie
%A Peng, Minlong
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F lin-etal-2026-beyond
%X Modern supervised fine-tuning (SFT) trains large language models on heterogeneous instruction mixtures, where the effective unit of optimization is an individual instruction-response pair rather than a coarse task label. Existing parameter isolation methods mitigate interference by assigning protected or trainable subspaces at the task or training stage level, but these units are often misaligned with actual parameter usage: instructions from the same task may require conflicting computations, while instructions from different tasks may share latent capabilities. We propose Instruction-Level Parameter Isolation (IPI), a fine-tuning framework that routes each instruction’s update to a sparse, capability-conditioned parameter subspace. IPI builds a multi-view instruction representation, predicts importance factors with a lightweight router, and uses an identity-trick reparameterization that preserves dense forward computation while exposing sparse base-model gradients and a task-loss saliency signal for router learning. Across heterogeneous instruction tuning benchmarks and multiple backbone models, IPI consistently improves performance over strong fine-tuning and parameter-isolation baselines. Further analyses show that instruction-conditioned masks reduce destructive gradient interactions and capture optimization structure that task labels miss.
%U https://aclanthology.org/2026.emnlp-main.287/
%P 6571-6595
Markdown (Informal)
[Beyond Task Boundaries: Instruction-Level Parameter Isolation for Supervised Fine-Tuning](https://aclanthology.org/2026.emnlp-main.287/) (Lin et al., EMNLP 2026)
ACL
- Zekai Lin, Long Xiang, Di Liang, Peiyang Liu, Zhanghao Hu, Mengze Hong, Xianjie Wu, and Minlong Peng. 2026. Beyond Task Boundaries: Instruction-Level Parameter Isolation for Supervised Fine-Tuning. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 6571–6595, Budapest, Hungary. Association for Computational Linguistics.