@inproceedings{chen-etal-2026-heterogeneous,
title = "Can Heterogeneous Language Models Be Fused?",
author = "Chen, Shilian and
Zhou, Jie and
Chen, Qin and
Wu, Wen and
Li, Xin and
Qi, Feng and
He, Liang",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.939/",
pages = "21650--21669",
ISBN = "979-8-89176-459-0",
abstract = "Model merging aims to integrate multiple expert models into a single model that inherits their complementary strengths without incurring the inference-time cost of ensembling. Recent progress has shown that merging can be highly effective when all source models are homogeneous, i.e., derived from the same pretrained backbone and therefore share aligned parameter coordinates or compatible task vectors. Yet this assumption is increasingly unrealistic in open model ecosystems, where useful experts are often built on different families such as Llama, Qwen, and Mistral. In such heterogeneous settings, direct weight-space fusion becomes ill-posed due to architectural mismatch, latent basis misalignment, and amplified cross-source conflict. We address this problem with HeteroFusion for heterogeneous language model fusion, which consists of two key components: topology-based alignment that transfers knowledge across heterogeneous backbones by matching functional module structures instead of raw tensor coordinates, and conflict-aware denoising that suppresses incompatible or noisy transfer signals during fusion, with a small replay set used to assist transfer-operator optimization. We further provide analytical and empirical evidence for preserving the target adapter basis while predicting structured updates. Across heterogeneous transfer, multi-source fusion, noisy-source robustness, and cross-family generalization settings, HeteroFusion consistently outperforms strong merging, fusion, and ensemble baselines."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chen-etal-2026-heterogeneous">
<titleInfo>
<title>Can Heterogeneous Language Models Be Fused?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shilian</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jie</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qin</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wen</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Feng</namePart>
<namePart type="family">Qi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Liang</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Model merging aims to integrate multiple expert models into a single model that inherits their complementary strengths without incurring the inference-time cost of ensembling. Recent progress has shown that merging can be highly effective when all source models are homogeneous, i.e., derived from the same pretrained backbone and therefore share aligned parameter coordinates or compatible task vectors. Yet this assumption is increasingly unrealistic in open model ecosystems, where useful experts are often built on different families such as Llama, Qwen, and Mistral. In such heterogeneous settings, direct weight-space fusion becomes ill-posed due to architectural mismatch, latent basis misalignment, and amplified cross-source conflict. We address this problem with HeteroFusion for heterogeneous language model fusion, which consists of two key components: topology-based alignment that transfers knowledge across heterogeneous backbones by matching functional module structures instead of raw tensor coordinates, and conflict-aware denoising that suppresses incompatible or noisy transfer signals during fusion, with a small replay set used to assist transfer-operator optimization. We further provide analytical and empirical evidence for preserving the target adapter basis while predicting structured updates. Across heterogeneous transfer, multi-source fusion, noisy-source robustness, and cross-family generalization settings, HeteroFusion consistently outperforms strong merging, fusion, and ensemble baselines.</abstract>
<identifier type="citekey">chen-etal-2026-heterogeneous</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.939/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>21650</start>
<end>21669</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Can Heterogeneous Language Models Be Fused?
%A Chen, Shilian
%A Zhou, Jie
%A Chen, Qin
%A Wu, Wen
%A Li, Xin
%A Qi, Feng
%A He, Liang
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F chen-etal-2026-heterogeneous
%X Model merging aims to integrate multiple expert models into a single model that inherits their complementary strengths without incurring the inference-time cost of ensembling. Recent progress has shown that merging can be highly effective when all source models are homogeneous, i.e., derived from the same pretrained backbone and therefore share aligned parameter coordinates or compatible task vectors. Yet this assumption is increasingly unrealistic in open model ecosystems, where useful experts are often built on different families such as Llama, Qwen, and Mistral. In such heterogeneous settings, direct weight-space fusion becomes ill-posed due to architectural mismatch, latent basis misalignment, and amplified cross-source conflict. We address this problem with HeteroFusion for heterogeneous language model fusion, which consists of two key components: topology-based alignment that transfers knowledge across heterogeneous backbones by matching functional module structures instead of raw tensor coordinates, and conflict-aware denoising that suppresses incompatible or noisy transfer signals during fusion, with a small replay set used to assist transfer-operator optimization. We further provide analytical and empirical evidence for preserving the target adapter basis while predicting structured updates. Across heterogeneous transfer, multi-source fusion, noisy-source robustness, and cross-family generalization settings, HeteroFusion consistently outperforms strong merging, fusion, and ensemble baselines.
%U https://aclanthology.org/2026.emnlp-main.939/
%P 21650-21669
Markdown (Informal)
[Can Heterogeneous Language Models Be Fused?](https://aclanthology.org/2026.emnlp-main.939/) (Chen et al., EMNLP 2026)
ACL
- Shilian Chen, Jie Zhou, Qin Chen, Wen Wu, Xin Li, Feng Qi, and Liang He. 2026. Can Heterogeneous Language Models Be Fused?. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 21650–21669, Budapest, Hungary. Association for Computational Linguistics.