@inproceedings{zhao-etal-2026-select,
title = "Select the Key, Then Generate the Rest: Improving Multi-Modal Learning with Limited Data Budget",
author = "Zhao, Xinyu and
Wang, Kaishen and
Yang, Jiaye and
Peng, Jie and
Xiang, Lirong and
Siddiqui, Faizan and
Lee, Kwonjoon and
Dariush, Behzad and
Chen, Tianlong",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.683/",
pages = "15727--15741",
ISBN = "979-8-89176-459-0",
abstract = "Multimodal learning serves as a promising approach for applications with diverse information sources. However, there are numerous challenges when scaling up multimodal learning data from all modalities due to availability or varied cost of data collection. We formulate and study multimodal learning under limited data collection budgets as a modality selection problem, demonstrating that selectively collecting a subset of modalities and synthesizing the missing ones can match or outperform models trained with uniformly augmented full modalities. Our research problem is formulated as: given a limited data collection budget, how to find the appropriate modalities to collect new data and generate for the rest to maximize model performance gain? To answer this, we propose a new paradigm - Select the Key modality, then generate the rest to enable learning with limited data (SK-ll). SK-ll contains two essential components: (1) Select the key. We propose a modality importance indicator to find the optimal modalities by assessing their single modal marginal contribution and cross-modal interactions. (2) Generate the rest. We substitute with generated embeddings for the rest of modalities. We conducted extensive experiments applying SK-ll across affective computing, healthcare with diverse multimodal learning backbones, obtaining average accuracy gains of 2.37{\%}, 6.55{\%}, 6.73{\%} on MOSI, MOSEI, ADNI respectively. Meanwhile, we present interesting empirical insights such as the data efficiency. Codes are provided in https://github.com/UNITES-Lab/SK-ll."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhao-etal-2026-select">
<titleInfo>
<title>Select the Key, Then Generate the Rest: Improving Multi-Modal Learning with Limited Data Budget</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kaishen</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiaye</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jie</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lirong</namePart>
<namePart type="family">Xiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Faizan</namePart>
<namePart type="family">Siddiqui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kwonjoon</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Behzad</namePart>
<namePart type="family">Dariush</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianlong</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Multimodal learning serves as a promising approach for applications with diverse information sources. However, there are numerous challenges when scaling up multimodal learning data from all modalities due to availability or varied cost of data collection. We formulate and study multimodal learning under limited data collection budgets as a modality selection problem, demonstrating that selectively collecting a subset of modalities and synthesizing the missing ones can match or outperform models trained with uniformly augmented full modalities. Our research problem is formulated as: given a limited data collection budget, how to find the appropriate modalities to collect new data and generate for the rest to maximize model performance gain? To answer this, we propose a new paradigm - Select the Key modality, then generate the rest to enable learning with limited data (SK-ll). SK-ll contains two essential components: (1) Select the key. We propose a modality importance indicator to find the optimal modalities by assessing their single modal marginal contribution and cross-modal interactions. (2) Generate the rest. We substitute with generated embeddings for the rest of modalities. We conducted extensive experiments applying SK-ll across affective computing, healthcare with diverse multimodal learning backbones, obtaining average accuracy gains of 2.37%, 6.55%, 6.73% on MOSI, MOSEI, ADNI respectively. Meanwhile, we present interesting empirical insights such as the data efficiency. Codes are provided in https://github.com/UNITES-Lab/SK-ll.</abstract>
<identifier type="citekey">zhao-etal-2026-select</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.683/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>15727</start>
<end>15741</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Select the Key, Then Generate the Rest: Improving Multi-Modal Learning with Limited Data Budget
%A Zhao, Xinyu
%A Wang, Kaishen
%A Yang, Jiaye
%A Peng, Jie
%A Xiang, Lirong
%A Siddiqui, Faizan
%A Lee, Kwonjoon
%A Dariush, Behzad
%A Chen, Tianlong
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F zhao-etal-2026-select
%X Multimodal learning serves as a promising approach for applications with diverse information sources. However, there are numerous challenges when scaling up multimodal learning data from all modalities due to availability or varied cost of data collection. We formulate and study multimodal learning under limited data collection budgets as a modality selection problem, demonstrating that selectively collecting a subset of modalities and synthesizing the missing ones can match or outperform models trained with uniformly augmented full modalities. Our research problem is formulated as: given a limited data collection budget, how to find the appropriate modalities to collect new data and generate for the rest to maximize model performance gain? To answer this, we propose a new paradigm - Select the Key modality, then generate the rest to enable learning with limited data (SK-ll). SK-ll contains two essential components: (1) Select the key. We propose a modality importance indicator to find the optimal modalities by assessing their single modal marginal contribution and cross-modal interactions. (2) Generate the rest. We substitute with generated embeddings for the rest of modalities. We conducted extensive experiments applying SK-ll across affective computing, healthcare with diverse multimodal learning backbones, obtaining average accuracy gains of 2.37%, 6.55%, 6.73% on MOSI, MOSEI, ADNI respectively. Meanwhile, we present interesting empirical insights such as the data efficiency. Codes are provided in https://github.com/UNITES-Lab/SK-ll.
%U https://aclanthology.org/2026.emnlp-main.683/
%P 15727-15741
Markdown (Informal)
[Select the Key, Then Generate the Rest: Improving Multi-Modal Learning with Limited Data Budget](https://aclanthology.org/2026.emnlp-main.683/) (Zhao et al., EMNLP 2026)
ACL
- Xinyu Zhao, Kaishen Wang, Jiaye Yang, Jie Peng, Lirong Xiang, Faizan Siddiqui, Kwonjoon Lee, Behzad Dariush, and Tianlong Chen. 2026. Select the Key, Then Generate the Rest: Improving Multi-Modal Learning with Limited Data Budget. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 15727–15741, Budapest, Hungary. Association for Computational Linguistics.