@inproceedings{oh-etal-2026-dynamic,
title = "Dynamic routing-based Mixture of Layer-wise Tokens with Token-Difficulty for Robust Audio-Visual Speech Recognition",
author = "Oh, Changhan and
Jo, Yeeun and
Han, Ran and
Kang, Jeomja and
Choi, Woo Yong and
Song, Hwa Jeon and
Park, Kiyoung",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1872/",
doi = "10.18653/2026.findings-emnlp.1872",
pages = "39954--39969",
ISBN = "979-8-89176-462-0",
abstract = "Audio-Visual Speech Recognition (AVSR) systems typically consume only the final-layer representation of their modality-specific encoders. In self-supervised (SSL) encoders, lower and upper layers encode acoustic and more abstract linguistic cues respectively, so the most informative layer may differ from token to token. We propose Mixture of Layer-wise Tokens (MoLT), which aggregates the per-layer outputs of the AV-HuBERT encoder per token through an MoE-style router. A sparse top-k MoLT (S-MoLT) selects the most useful layers per token, and Dynamic routing-based MoLT (D-MoLT) replaces the fixed k with a learnable per-token threshold. To align routing with recognition difficulty, we further add a token-difficulty-aware lower-upper preference that biases, rather than overrides, the learned routing: Difficulty-adaptive D-MoLT (DD-MoLT) derives the difficulty signal from one modality, while Cross-modal DD-MoLT (CDD-MoLT) uses the audio-video reliability gap. On LRS3, our models consistently improve WER over a strong MMS-LLaMA baseline across SNR conditions, with the gain widening under severe noise (e.g., 13.85 vs. 17.10{\%} at{\ensuremath{-}}15 dB), at negligible overhead."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="oh-etal-2026-dynamic">
<titleInfo>
<title>Dynamic routing-based Mixture of Layer-wise Tokens with Token-Difficulty for Robust Audio-Visual Speech Recognition</title>
</titleInfo>
<name type="personal">
<namePart type="given">Changhan</namePart>
<namePart type="family">Oh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yeeun</namePart>
<namePart type="family">Jo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ran</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jeomja</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Woo</namePart>
<namePart type="given">Yong</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hwa</namePart>
<namePart type="given">Jeon</namePart>
<namePart type="family">Song</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kiyoung</namePart>
<namePart type="family">Park</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Audio-Visual Speech Recognition (AVSR) systems typically consume only the final-layer representation of their modality-specific encoders. In self-supervised (SSL) encoders, lower and upper layers encode acoustic and more abstract linguistic cues respectively, so the most informative layer may differ from token to token. We propose Mixture of Layer-wise Tokens (MoLT), which aggregates the per-layer outputs of the AV-HuBERT encoder per token through an MoE-style router. A sparse top-k MoLT (S-MoLT) selects the most useful layers per token, and Dynamic routing-based MoLT (D-MoLT) replaces the fixed k with a learnable per-token threshold. To align routing with recognition difficulty, we further add a token-difficulty-aware lower-upper preference that biases, rather than overrides, the learned routing: Difficulty-adaptive D-MoLT (DD-MoLT) derives the difficulty signal from one modality, while Cross-modal DD-MoLT (CDD-MoLT) uses the audio-video reliability gap. On LRS3, our models consistently improve WER over a strong MMS-LLaMA baseline across SNR conditions, with the gain widening under severe noise (e.g., 13.85 vs. 17.10% at\ensuremath-15 dB), at negligible overhead.</abstract>
<identifier type="citekey">oh-etal-2026-dynamic</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1872</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1872/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>39954</start>
<end>39969</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Dynamic routing-based Mixture of Layer-wise Tokens with Token-Difficulty for Robust Audio-Visual Speech Recognition
%A Oh, Changhan
%A Jo, Yeeun
%A Han, Ran
%A Kang, Jeomja
%A Choi, Woo Yong
%A Song, Hwa Jeon
%A Park, Kiyoung
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F oh-etal-2026-dynamic
%X Audio-Visual Speech Recognition (AVSR) systems typically consume only the final-layer representation of their modality-specific encoders. In self-supervised (SSL) encoders, lower and upper layers encode acoustic and more abstract linguistic cues respectively, so the most informative layer may differ from token to token. We propose Mixture of Layer-wise Tokens (MoLT), which aggregates the per-layer outputs of the AV-HuBERT encoder per token through an MoE-style router. A sparse top-k MoLT (S-MoLT) selects the most useful layers per token, and Dynamic routing-based MoLT (D-MoLT) replaces the fixed k with a learnable per-token threshold. To align routing with recognition difficulty, we further add a token-difficulty-aware lower-upper preference that biases, rather than overrides, the learned routing: Difficulty-adaptive D-MoLT (DD-MoLT) derives the difficulty signal from one modality, while Cross-modal DD-MoLT (CDD-MoLT) uses the audio-video reliability gap. On LRS3, our models consistently improve WER over a strong MMS-LLaMA baseline across SNR conditions, with the gain widening under severe noise (e.g., 13.85 vs. 17.10% at\ensuremath-15 dB), at negligible overhead.
%R 10.18653/2026.findings-emnlp.1872
%U https://aclanthology.org/2026.findings-emnlp.1872/
%U https://doi.org/10.18653/2026.findings-emnlp.1872
%P 39954-39969
Markdown (Informal)
[Dynamic routing-based Mixture of Layer-wise Tokens with Token-Difficulty for Robust Audio-Visual Speech Recognition](https://aclanthology.org/2026.findings-emnlp.1872/) (Oh et al., Findings 2026)
ACL