@inproceedings{burenko-etal-2026-automatic,
title = "Automatic Annotation of Co-speech Gesture for Multimodal Communication Research {--} A Framework and a Case Study",
author = "Burenko, Ilya and
Uhrig, Peter and
Wilson, Anna",
editor = "Ghaleb, Esam and
Takmaz, Ece and
Frassinelli, Diego and
Khaertdinov, Bulat and
Fern{\'a}ndez, Raquel and
Yumak, Zerrin and
Ozyurek, Asli",
booktitle = "Proceedings of the 1st Workshop on Multimodal Interaction in Face-to-Face Dialogue",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mint-1.7/",
pages = "59--71",
ISBN = "979-8-89176-482-8",
abstract = "Large-scale multimodal communication research increasingly depends on computational methods capable of extracting reliable gesture annotations from video data. Yet it remains unclear to what extent automatically generated annotations can support the linguistic investigation of naturally occurring co-speech gesture and where expert interpretation remains indispensable. This paper presents a framework for the automatic analysis and annotation of co-speech gesture in media corpora, developed to support multimodal communication research at scale. We build our framework on top of the existing computer vision frameworks for keypoint extraction and active speaker detection, with post-processing steps carefully designed to mitigate noisy output. Building on the processed keypoint trajectories, we implement a configurable rule-based annotation system developed through close collaboration between computer vision researchers and linguists. We demonstrate the usefulness of the framework through a case study investigating vertical hand movement accompanying English expressions of future and past, examining which aspects of this gestural pattern can be captured through automatic annotation and which require expert multimodal interpretation. We find that at corpus scale, greater mean amplitude and higher mean peak velocity converge in *will*-future, suggesting greater kinematic prominence in the multimodal expression of futurity. More broadly, the case study demonstrates how computational pipelines can support, rather than replace, human analysis by enabling multimodal hypotheses to be investigated on datasets that would be impractical to analyse manually."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="burenko-etal-2026-automatic">
<titleInfo>
<title>Automatic Annotation of Co-speech Gesture for Multimodal Communication Research – A Framework and a Case Study</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ilya</namePart>
<namePart type="family">Burenko</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peter</namePart>
<namePart type="family">Uhrig</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="family">Wilson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Multimodal Interaction in Face-to-Face Dialogue</title>
</titleInfo>
<name type="personal">
<namePart type="given">Esam</namePart>
<namePart type="family">Ghaleb</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ece</namePart>
<namePart type="family">Takmaz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Diego</namePart>
<namePart type="family">Frassinelli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bulat</namePart>
<namePart type="family">Khaertdinov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raquel</namePart>
<namePart type="family">Fernández</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zerrin</namePart>
<namePart type="family">Yumak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Asli</namePart>
<namePart type="family">Ozyurek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-482-8</identifier>
</relatedItem>
<abstract>Large-scale multimodal communication research increasingly depends on computational methods capable of extracting reliable gesture annotations from video data. Yet it remains unclear to what extent automatically generated annotations can support the linguistic investigation of naturally occurring co-speech gesture and where expert interpretation remains indispensable. This paper presents a framework for the automatic analysis and annotation of co-speech gesture in media corpora, developed to support multimodal communication research at scale. We build our framework on top of the existing computer vision frameworks for keypoint extraction and active speaker detection, with post-processing steps carefully designed to mitigate noisy output. Building on the processed keypoint trajectories, we implement a configurable rule-based annotation system developed through close collaboration between computer vision researchers and linguists. We demonstrate the usefulness of the framework through a case study investigating vertical hand movement accompanying English expressions of future and past, examining which aspects of this gestural pattern can be captured through automatic annotation and which require expert multimodal interpretation. We find that at corpus scale, greater mean amplitude and higher mean peak velocity converge in *will*-future, suggesting greater kinematic prominence in the multimodal expression of futurity. More broadly, the case study demonstrates how computational pipelines can support, rather than replace, human analysis by enabling multimodal hypotheses to be investigated on datasets that would be impractical to analyse manually.</abstract>
<identifier type="citekey">burenko-etal-2026-automatic</identifier>
<location>
<url>https://aclanthology.org/2026.mint-1.7/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>59</start>
<end>71</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Automatic Annotation of Co-speech Gesture for Multimodal Communication Research – A Framework and a Case Study
%A Burenko, Ilya
%A Uhrig, Peter
%A Wilson, Anna
%Y Ghaleb, Esam
%Y Takmaz, Ece
%Y Frassinelli, Diego
%Y Khaertdinov, Bulat
%Y Fernández, Raquel
%Y Yumak, Zerrin
%Y Ozyurek, Asli
%S Proceedings of the 1st Workshop on Multimodal Interaction in Face-to-Face Dialogue
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-482-8
%F burenko-etal-2026-automatic
%X Large-scale multimodal communication research increasingly depends on computational methods capable of extracting reliable gesture annotations from video data. Yet it remains unclear to what extent automatically generated annotations can support the linguistic investigation of naturally occurring co-speech gesture and where expert interpretation remains indispensable. This paper presents a framework for the automatic analysis and annotation of co-speech gesture in media corpora, developed to support multimodal communication research at scale. We build our framework on top of the existing computer vision frameworks for keypoint extraction and active speaker detection, with post-processing steps carefully designed to mitigate noisy output. Building on the processed keypoint trajectories, we implement a configurable rule-based annotation system developed through close collaboration between computer vision researchers and linguists. We demonstrate the usefulness of the framework through a case study investigating vertical hand movement accompanying English expressions of future and past, examining which aspects of this gestural pattern can be captured through automatic annotation and which require expert multimodal interpretation. We find that at corpus scale, greater mean amplitude and higher mean peak velocity converge in *will*-future, suggesting greater kinematic prominence in the multimodal expression of futurity. More broadly, the case study demonstrates how computational pipelines can support, rather than replace, human analysis by enabling multimodal hypotheses to be investigated on datasets that would be impractical to analyse manually.
%U https://aclanthology.org/2026.mint-1.7/
%P 59-71
Markdown (Informal)
[Automatic Annotation of Co-speech Gesture for Multimodal Communication Research – A Framework and a Case Study](https://aclanthology.org/2026.mint-1.7/) (Burenko et al., MINT 2026)
ACL