@inproceedings{dai-etal-2026-plan,
title = "Plan the Semantics, Then Render the Signal: A Hybrid State-Space Model for Incomplete Multimodal Learning",
author = "Dai, Ruiting and
Qin, Zhihao and
Chen, Yilin and
Yan, Xianhua and
Mo, Lisi and
He, Tao",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1563/",
doi = "10.18653/2026.findings-emnlp.1563",
pages = "33292--33308",
ISBN = "979-8-89176-462-0",
abstract = "Incomplete multimodal learning seeks robust perception when modalities are absent at training or test time, using available streams to recover the missing ones. Most methods cast this as direct regression in a fixed continuous latent, which (i) collapses many valid solutions into an averaged reconstruction when observations underdetermine the target, and (ii) suffers semantic drift when cross-modal relationships shift with context. We argue that completion should first resolve what is missing and only then render how it appears. We realize this as Semantic-first Completion, instantiated by a Hybrid State-Space Model (HySSM) that decouples semantic intent from signal realization. A frozen omni-modal LLM plans over observation-only evidence, generating an evidence-grounded Markovian trajectory of discrete semantic states that act as a compact information bottleneck for stable cross-modal reasoning. The plan is projected into a coordinate-wise control field that drift-guides a conditional diffusion model, synthesizing missing signals or feature streams as diverse and semantically consistent completions. Across MNIST, Hateful Memes, and CMU-MOSEI, HySSM surpasses recent generative baselines and, under agnostic masking shift, remains robust up to 90{\%} missingness."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="dai-etal-2026-plan">
<titleInfo>
<title>Plan the Semantics, Then Render the Signal: A Hybrid State-Space Model for Incomplete Multimodal Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ruiting</namePart>
<namePart type="family">Dai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhihao</namePart>
<namePart type="family">Qin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yilin</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xianhua</namePart>
<namePart type="family">Yan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lisi</namePart>
<namePart type="family">Mo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tao</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Incomplete multimodal learning seeks robust perception when modalities are absent at training or test time, using available streams to recover the missing ones. Most methods cast this as direct regression in a fixed continuous latent, which (i) collapses many valid solutions into an averaged reconstruction when observations underdetermine the target, and (ii) suffers semantic drift when cross-modal relationships shift with context. We argue that completion should first resolve what is missing and only then render how it appears. We realize this as Semantic-first Completion, instantiated by a Hybrid State-Space Model (HySSM) that decouples semantic intent from signal realization. A frozen omni-modal LLM plans over observation-only evidence, generating an evidence-grounded Markovian trajectory of discrete semantic states that act as a compact information bottleneck for stable cross-modal reasoning. The plan is projected into a coordinate-wise control field that drift-guides a conditional diffusion model, synthesizing missing signals or feature streams as diverse and semantically consistent completions. Across MNIST, Hateful Memes, and CMU-MOSEI, HySSM surpasses recent generative baselines and, under agnostic masking shift, remains robust up to 90% missingness.</abstract>
<identifier type="citekey">dai-etal-2026-plan</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1563</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1563/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>33292</start>
<end>33308</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Plan the Semantics, Then Render the Signal: A Hybrid State-Space Model for Incomplete Multimodal Learning
%A Dai, Ruiting
%A Qin, Zhihao
%A Chen, Yilin
%A Yan, Xianhua
%A Mo, Lisi
%A He, Tao
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F dai-etal-2026-plan
%X Incomplete multimodal learning seeks robust perception when modalities are absent at training or test time, using available streams to recover the missing ones. Most methods cast this as direct regression in a fixed continuous latent, which (i) collapses many valid solutions into an averaged reconstruction when observations underdetermine the target, and (ii) suffers semantic drift when cross-modal relationships shift with context. We argue that completion should first resolve what is missing and only then render how it appears. We realize this as Semantic-first Completion, instantiated by a Hybrid State-Space Model (HySSM) that decouples semantic intent from signal realization. A frozen omni-modal LLM plans over observation-only evidence, generating an evidence-grounded Markovian trajectory of discrete semantic states that act as a compact information bottleneck for stable cross-modal reasoning. The plan is projected into a coordinate-wise control field that drift-guides a conditional diffusion model, synthesizing missing signals or feature streams as diverse and semantically consistent completions. Across MNIST, Hateful Memes, and CMU-MOSEI, HySSM surpasses recent generative baselines and, under agnostic masking shift, remains robust up to 90% missingness.
%R 10.18653/2026.findings-emnlp.1563
%U https://aclanthology.org/2026.findings-emnlp.1563/
%U https://doi.org/10.18653/2026.findings-emnlp.1563
%P 33292-33308
Markdown (Informal)
[Plan the Semantics, Then Render the Signal: A Hybrid State-Space Model for Incomplete Multimodal Learning](https://aclanthology.org/2026.findings-emnlp.1563/) (Dai et al., Findings 2026)
ACL