@inproceedings{poppi-etal-2026-countervid,
title = "{C}ounter{V}id: Counterfactual Video Generation for Mitigating Action and Temporal Hallucinations in Video-Language Models",
author = "Poppi, Tobia and
Uzkent, Burak and
Garg, Amanmeet and
Porto, Lucas Resque and
Kessler, Garin N. and
Yang, YZ and
Cornia, Marcella and
Baraldi, Lorenzo and
Cucchiara, Rita and
Schiffers, Florian",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1389/",
pages = "31712--31731",
ISBN = "979-8-89176-459-0",
abstract = "Video-language models (VLMs) achieve strong multimodal understanding but remain prone to hallucinations, especially when reasoning about actions and temporal order. Existing mitigation strategies, such as textual filtering or random video perturbations, often fail to address the root cause: over-reliance on language priors rather than fine-grained visual dynamics. We propose a scalable framework for counterfactual video generation that synthesizes videos differing only in actions or temporal structure while preserving scene context. Our pipeline combines multimodal LLMs for action proposal and editing guidance with diffusion-based image and video models to generate semantic hard negatives at scale. Using this framework, we build CounterVid, a synthetic dataset of {\textasciitilde}26k preference pairs constructed from short counterfactual action clips and targeting both action recognition and controlled action-sequence ordering. We further introduce MixDPO, a unified Direct Preference Optimization approach that jointly leverages textual and visual preferences. Across Qwen2.5-VL and InternVL3 backbones, MixDPO substantially improves action recognition and temporal ordering, yields gains on most standard video hallucination benchmarks, and largely preserves general video understanding. Our source code, trained models, and dataset are available at https://aimagelab.github.io/CounterVid."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="poppi-etal-2026-countervid">
<titleInfo>
<title>CounterVid: Counterfactual Video Generation for Mitigating Action and Temporal Hallucinations in Video-Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Tobia</namePart>
<namePart type="family">Poppi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Burak</namePart>
<namePart type="family">Uzkent</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Amanmeet</namePart>
<namePart type="family">Garg</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lucas</namePart>
<namePart type="given">Resque</namePart>
<namePart type="family">Porto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Garin</namePart>
<namePart type="given">N</namePart>
<namePart type="family">Kessler</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Y</namePart>
<namePart type="given">Z</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marcella</namePart>
<namePart type="family">Cornia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lorenzo</namePart>
<namePart type="family">Baraldi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rita</namePart>
<namePart type="family">Cucchiara</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Florian</namePart>
<namePart type="family">Schiffers</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Video-language models (VLMs) achieve strong multimodal understanding but remain prone to hallucinations, especially when reasoning about actions and temporal order. Existing mitigation strategies, such as textual filtering or random video perturbations, often fail to address the root cause: over-reliance on language priors rather than fine-grained visual dynamics. We propose a scalable framework for counterfactual video generation that synthesizes videos differing only in actions or temporal structure while preserving scene context. Our pipeline combines multimodal LLMs for action proposal and editing guidance with diffusion-based image and video models to generate semantic hard negatives at scale. Using this framework, we build CounterVid, a synthetic dataset of ~26k preference pairs constructed from short counterfactual action clips and targeting both action recognition and controlled action-sequence ordering. We further introduce MixDPO, a unified Direct Preference Optimization approach that jointly leverages textual and visual preferences. Across Qwen2.5-VL and InternVL3 backbones, MixDPO substantially improves action recognition and temporal ordering, yields gains on most standard video hallucination benchmarks, and largely preserves general video understanding. Our source code, trained models, and dataset are available at https://aimagelab.github.io/CounterVid.</abstract>
<identifier type="citekey">poppi-etal-2026-countervid</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1389/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>31712</start>
<end>31731</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T CounterVid: Counterfactual Video Generation for Mitigating Action and Temporal Hallucinations in Video-Language Models
%A Poppi, Tobia
%A Uzkent, Burak
%A Garg, Amanmeet
%A Porto, Lucas Resque
%A Kessler, Garin N.
%A Yang, Y. Z.
%A Cornia, Marcella
%A Baraldi, Lorenzo
%A Cucchiara, Rita
%A Schiffers, Florian
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F poppi-etal-2026-countervid
%X Video-language models (VLMs) achieve strong multimodal understanding but remain prone to hallucinations, especially when reasoning about actions and temporal order. Existing mitigation strategies, such as textual filtering or random video perturbations, often fail to address the root cause: over-reliance on language priors rather than fine-grained visual dynamics. We propose a scalable framework for counterfactual video generation that synthesizes videos differing only in actions or temporal structure while preserving scene context. Our pipeline combines multimodal LLMs for action proposal and editing guidance with diffusion-based image and video models to generate semantic hard negatives at scale. Using this framework, we build CounterVid, a synthetic dataset of ~26k preference pairs constructed from short counterfactual action clips and targeting both action recognition and controlled action-sequence ordering. We further introduce MixDPO, a unified Direct Preference Optimization approach that jointly leverages textual and visual preferences. Across Qwen2.5-VL and InternVL3 backbones, MixDPO substantially improves action recognition and temporal ordering, yields gains on most standard video hallucination benchmarks, and largely preserves general video understanding. Our source code, trained models, and dataset are available at https://aimagelab.github.io/CounterVid.
%U https://aclanthology.org/2026.emnlp-main.1389/
%P 31712-31731
Markdown (Informal)
[CounterVid: Counterfactual Video Generation for Mitigating Action and Temporal Hallucinations in Video-Language Models](https://aclanthology.org/2026.emnlp-main.1389/) (Poppi et al., EMNLP 2026)
ACL
- Tobia Poppi, Burak Uzkent, Amanmeet Garg, Lucas Resque Porto, Garin N. Kessler, YZ Yang, Marcella Cornia, Lorenzo Baraldi, Rita Cucchiara, and Florian Schiffers. 2026. CounterVid: Counterfactual Video Generation for Mitigating Action and Temporal Hallucinations in Video-Language Models. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 31712–31731, Budapest, Hungary. Association for Computational Linguistics.