@inproceedings{jiang-etal-2026-cofie,
title = "{C}o{F}i{E}: Coarse-to-Fine Evidence Selection for Efficient Streaming Video Understanding",
author = "Jiang, Jing and
Ling, Yiran and
Li, Ruonan and
Stamoulis, Dimitrios and
Liu, Jie",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.518/",
pages = "11893--11905",
ISBN = "979-8-89176-459-0",
abstract = "Streaming video understanding requires Vision Language Models (VLLMs) to process growing video streams and answer user questions under tight latency constraints. Existing methods improve efficiency with token pruning and memory-bank schemes, but mainly reduce visual tokens $\textit{after}$ the LLM encoding stage. Therefore, downstream token pruning alone cannot substantially reduce end-to-end latency, since the expensive frame encoding cost has already been incurred. In this work, we propose CoFiE, a $\textbf{Co}$arse-to-$\textbf{Fi}$ne $\textbf{E}$vidence selection framework, that decouples evidence selection into a $\textit{coarse}$ query-agnostic filtering stage before the vision encoder and a $\textit{fine}$ query-specific refinement stage during LLM prefill. We introduce two lightweight modules, namely the coarse Novelty-Guided Frame Filtering and the fine Query-Specific Evidence Refinement, that retain visually distinctive candidates and refine the most relevant frames, respectively. This design removes substantial redundancy before frame encoding while retaining query-specific refinement once semantic information becomes available. Experimental results show that CoFiE establishes a new state-of-the-art accuracy{--}efficiency trade-off with open-source VLLMs across multiple video understanding benchmarks, reaching 78.86{\%} on StreamingBench and 68.72{\%} on OvO-Bench, improving over prior best methods by up to 3.15{\%}. Notably, even under aggressive evidence-frame filtering up to 80{\%}, CoFiE outperforms open-source frontier multimodal models while improving end-to-end inference latency by up to 2.54$\times$."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="jiang-etal-2026-cofie">
<titleInfo>
<title>CoFiE: Coarse-to-Fine Evidence Selection for Efficient Streaming Video Understanding</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jing</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yiran</namePart>
<namePart type="family">Ling</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruonan</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dimitrios</namePart>
<namePart type="family">Stamoulis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jie</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Streaming video understanding requires Vision Language Models (VLLMs) to process growing video streams and answer user questions under tight latency constraints. Existing methods improve efficiency with token pruning and memory-bank schemes, but mainly reduce visual tokens after the LLM encoding stage. Therefore, downstream token pruning alone cannot substantially reduce end-to-end latency, since the expensive frame encoding cost has already been incurred. In this work, we propose CoFiE, a Coarse-to-Fine Evidence selection framework, that decouples evidence selection into a coarse query-agnostic filtering stage before the vision encoder and a fine query-specific refinement stage during LLM prefill. We introduce two lightweight modules, namely the coarse Novelty-Guided Frame Filtering and the fine Query-Specific Evidence Refinement, that retain visually distinctive candidates and refine the most relevant frames, respectively. This design removes substantial redundancy before frame encoding while retaining query-specific refinement once semantic information becomes available. Experimental results show that CoFiE establishes a new state-of-the-art accuracy–efficiency trade-off with open-source VLLMs across multiple video understanding benchmarks, reaching 78.86% on StreamingBench and 68.72% on OvO-Bench, improving over prior best methods by up to 3.15%. Notably, even under aggressive evidence-frame filtering up to 80%, CoFiE outperforms open-source frontier multimodal models while improving end-to-end inference latency by up to 2.54\times.</abstract>
<identifier type="citekey">jiang-etal-2026-cofie</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.518/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>11893</start>
<end>11905</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T CoFiE: Coarse-to-Fine Evidence Selection for Efficient Streaming Video Understanding
%A Jiang, Jing
%A Ling, Yiran
%A Li, Ruonan
%A Stamoulis, Dimitrios
%A Liu, Jie
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F jiang-etal-2026-cofie
%X Streaming video understanding requires Vision Language Models (VLLMs) to process growing video streams and answer user questions under tight latency constraints. Existing methods improve efficiency with token pruning and memory-bank schemes, but mainly reduce visual tokens after the LLM encoding stage. Therefore, downstream token pruning alone cannot substantially reduce end-to-end latency, since the expensive frame encoding cost has already been incurred. In this work, we propose CoFiE, a Coarse-to-Fine Evidence selection framework, that decouples evidence selection into a coarse query-agnostic filtering stage before the vision encoder and a fine query-specific refinement stage during LLM prefill. We introduce two lightweight modules, namely the coarse Novelty-Guided Frame Filtering and the fine Query-Specific Evidence Refinement, that retain visually distinctive candidates and refine the most relevant frames, respectively. This design removes substantial redundancy before frame encoding while retaining query-specific refinement once semantic information becomes available. Experimental results show that CoFiE establishes a new state-of-the-art accuracy–efficiency trade-off with open-source VLLMs across multiple video understanding benchmarks, reaching 78.86% on StreamingBench and 68.72% on OvO-Bench, improving over prior best methods by up to 3.15%. Notably, even under aggressive evidence-frame filtering up to 80%, CoFiE outperforms open-source frontier multimodal models while improving end-to-end inference latency by up to 2.54\times.
%U https://aclanthology.org/2026.emnlp-main.518/
%P 11893-11905
Markdown (Informal)
[CoFiE: Coarse-to-Fine Evidence Selection for Efficient Streaming Video Understanding](https://aclanthology.org/2026.emnlp-main.518/) (Jiang et al., EMNLP 2026)
ACL