@inproceedings{kruengkrai-yoshino-2026-changes,
title = "What Changes in Hidden States When Model Merging Hurts Task Performance?",
author = "Kruengkrai, Canasai and
Yoshino, Koichiro",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.180/",
pages = "4021--4034",
ISBN = "979-8-89176-439-2",
abstract = "Model merging is a practical way to combine several fine-tuned models, but merged models can lose performance relative to their task-specific counterparts. Task accuracy shows the degradation, but not how merging changes the model{'}s internal representations. We study this question through hidden-state distortion, which measures the difference between a merged model{'}s hidden states and those of the corresponding task-specific reference model on the same examples. Across two model families, six tasks, and three merge methods, we find that the distortion is highly anisotropic: much of the merged-reference change is concentrated in a small number of directions. Removing distortion components along dominant directions often improves accuracy more than removing those along unrelated directions. When we jointly fit a single distortion basis across tasks for each model family, this removal still improves accuracy, suggesting that the tasks share some useful distortion structure. In addition, the distortion score provides a label-free criterion for choosing among candidate task-arithmetic merge scales, matching supervised selection for both model families."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kruengkrai-yoshino-2026-changes">
<titleInfo>
<title>What Changes in Hidden States When Model Merging Hurts Task Performance?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Canasai</namePart>
<namePart type="family">Kruengkrai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Koichiro</namePart>
<namePart type="family">Yoshino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Model merging is a practical way to combine several fine-tuned models, but merged models can lose performance relative to their task-specific counterparts. Task accuracy shows the degradation, but not how merging changes the model’s internal representations. We study this question through hidden-state distortion, which measures the difference between a merged model’s hidden states and those of the corresponding task-specific reference model on the same examples. Across two model families, six tasks, and three merge methods, we find that the distortion is highly anisotropic: much of the merged-reference change is concentrated in a small number of directions. Removing distortion components along dominant directions often improves accuracy more than removing those along unrelated directions. When we jointly fit a single distortion basis across tasks for each model family, this removal still improves accuracy, suggesting that the tasks share some useful distortion structure. In addition, the distortion score provides a label-free criterion for choosing among candidate task-arithmetic merge scales, matching supervised selection for both model families.</abstract>
<identifier type="citekey">kruengkrai-yoshino-2026-changes</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.180/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>4021</start>
<end>4034</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What Changes in Hidden States When Model Merging Hurts Task Performance?
%A Kruengkrai, Canasai
%A Yoshino, Koichiro
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F kruengkrai-yoshino-2026-changes
%X Model merging is a practical way to combine several fine-tuned models, but merged models can lose performance relative to their task-specific counterparts. Task accuracy shows the degradation, but not how merging changes the model’s internal representations. We study this question through hidden-state distortion, which measures the difference between a merged model’s hidden states and those of the corresponding task-specific reference model on the same examples. Across two model families, six tasks, and three merge methods, we find that the distortion is highly anisotropic: much of the merged-reference change is concentrated in a small number of directions. Removing distortion components along dominant directions often improves accuracy more than removing those along unrelated directions. When we jointly fit a single distortion basis across tasks for each model family, this removal still improves accuracy, suggesting that the tasks share some useful distortion structure. In addition, the distortion score provides a label-free criterion for choosing among candidate task-arithmetic merge scales, matching supervised selection for both model families.
%U https://aclanthology.org/2026.aacl-long.180/
%P 4021-4034
Markdown (Informal)
[What Changes in Hidden States When Model Merging Hurts Task Performance?](https://aclanthology.org/2026.aacl-long.180/) (Kruengkrai & Yoshino, AACL-IJCNLP 2026)
ACL
- Canasai Kruengkrai and Koichiro Yoshino. 2026. What Changes in Hidden States When Model Merging Hurts Task Performance?. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 4021–4034, Hengqin, China. Association for Computational Linguistics.