@inproceedings{koneru-etal-2026-omnifusion,
title = "{O}mni{F}usion: Simultaneous Multilingual and Multimodal Translations via Modular Fusion",
author = "Koneru, Sai and
Huck, Matthias and
Niehues, Jan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.320/",
pages = "6770--6788",
ISBN = "979-8-89176-462-0",
abstract = "Open-source text-only translation LLMs have rapidly improved multilingual coverage and translation quality, but their unimodal design limits their use in multimodal translation. In speech translation, they are commonly used in cascaded ASR{--}MT pipelines, which introduce additional latency and prevent the translation model from directly using visual context. Conversely, pretrained multimodal foundation models provide strong audio-visual perception but generally lag behind specialized translation LLMs in multilingual generation quality. We introduce OmniFusion, a modular architecture that connects a frozen multimodal foundation model to a translation-specialized LLM through a lightweight gated fusion module. OmniFusion supports speech-to-text, speech-and-image-to-text, and text-and-image-to-text translation within a single framework. Experiments with Qwen2.5-Omni-7B and Seed-X-PPO-7B show that selected early and middle MMFM layers provide more transferable representations than late layers. On MCIF, OmniFusion improves the latency{--}quality trade-off over a matched cascaded SimulST pipeline when evaluated with Local Agreement decoding, while remaining competitive in offline ST and reducing severe XCOMET-XL errors. On CoMMuTE, it achieves competitive visually grounded caption translation performance."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="koneru-etal-2026-omnifusion">
<titleInfo>
<title>OmniFusion: Simultaneous Multilingual and Multimodal Translations via Modular Fusion</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sai</namePart>
<namePart type="family">Koneru</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Matthias</namePart>
<namePart type="family">Huck</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jan</namePart>
<namePart type="family">Niehues</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Open-source text-only translation LLMs have rapidly improved multilingual coverage and translation quality, but their unimodal design limits their use in multimodal translation. In speech translation, they are commonly used in cascaded ASR–MT pipelines, which introduce additional latency and prevent the translation model from directly using visual context. Conversely, pretrained multimodal foundation models provide strong audio-visual perception but generally lag behind specialized translation LLMs in multilingual generation quality. We introduce OmniFusion, a modular architecture that connects a frozen multimodal foundation model to a translation-specialized LLM through a lightweight gated fusion module. OmniFusion supports speech-to-text, speech-and-image-to-text, and text-and-image-to-text translation within a single framework. Experiments with Qwen2.5-Omni-7B and Seed-X-PPO-7B show that selected early and middle MMFM layers provide more transferable representations than late layers. On MCIF, OmniFusion improves the latency–quality trade-off over a matched cascaded SimulST pipeline when evaluated with Local Agreement decoding, while remaining competitive in offline ST and reducing severe XCOMET-XL errors. On CoMMuTE, it achieves competitive visually grounded caption translation performance.</abstract>
<identifier type="citekey">koneru-etal-2026-omnifusion</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.320/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>6770</start>
<end>6788</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T OmniFusion: Simultaneous Multilingual and Multimodal Translations via Modular Fusion
%A Koneru, Sai
%A Huck, Matthias
%A Niehues, Jan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F koneru-etal-2026-omnifusion
%X Open-source text-only translation LLMs have rapidly improved multilingual coverage and translation quality, but their unimodal design limits their use in multimodal translation. In speech translation, they are commonly used in cascaded ASR–MT pipelines, which introduce additional latency and prevent the translation model from directly using visual context. Conversely, pretrained multimodal foundation models provide strong audio-visual perception but generally lag behind specialized translation LLMs in multilingual generation quality. We introduce OmniFusion, a modular architecture that connects a frozen multimodal foundation model to a translation-specialized LLM through a lightweight gated fusion module. OmniFusion supports speech-to-text, speech-and-image-to-text, and text-and-image-to-text translation within a single framework. Experiments with Qwen2.5-Omni-7B and Seed-X-PPO-7B show that selected early and middle MMFM layers provide more transferable representations than late layers. On MCIF, OmniFusion improves the latency–quality trade-off over a matched cascaded SimulST pipeline when evaluated with Local Agreement decoding, while remaining competitive in offline ST and reducing severe XCOMET-XL errors. On CoMMuTE, it achieves competitive visually grounded caption translation performance.
%U https://aclanthology.org/2026.findings-emnlp.320/
%P 6770-6788
Markdown (Informal)
[OmniFusion: Simultaneous Multilingual and Multimodal Translations via Modular Fusion](https://aclanthology.org/2026.findings-emnlp.320/) (Koneru et al., Findings 2026)
ACL