@inproceedings{gandhi-etal-2026-motions,
title = "Same Motions, Different Winners: The Gesture-Generation Leaderboard and Its Human Agreement Are Configuration-Tunable",
author = "Gandhi, Deep and
Asaria, Ali and
Salomone, Tony",
editor = "Ghaleb, Esam and
Takmaz, Ece and
Frassinelli, Diego and
Khaertdinov, Bulat and
Fern{\'a}ndez, Raquel and
Yumak, Zerrin and
Ozyurek, Asli",
booktitle = "Proceedings of the 1st Workshop on Multimodal Interaction in Face-to-Face Dialogue",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mint-1.8/",
pages = "72--77",
ISBN = "979-8-89176-482-8",
abstract = "Fr{\'e}chet Gesture Distance (FGD) is the standard automatic metric for co-speech gesture generation, the models that synthesize the hand and body motion accompanying speech. Human perceptual studies remain the gold standard, and the GENEA challenges rank systems by them, but such studies are slow and costly, so papers routinely compare systems by FGD instead. FGD is a Fr{\'e}chet distance in the latent space of a learned pose autoencoder, so it inherits the many choices behind that encoder, most of which papers never report. We ask whether FGD is a reliable proxy for perceptual evaluation under these unreported choices. Holding the systems fixed, we vary the reference tool{'}s own settings (analysis window, seed, feature versus raw space) and the corpus the feature extractor is trained on, on both tiers of the GENEA 2022 evaluation and on the GENEA 2023 challenge. First, the FGD ranking is not reproducible where it matters: the ``best'' system is one of two that a per-clip bootstrap shows are statistically tied, so which one wins is a configuration choice, though aggregate rank correlation stays moderate rather than collapsing. Second, and more consequential, FGD{'}s agreement with human ratings is modest and itself tunable, moving by 0.18 to 0.24 Kendall {\ensuremath{\tau}} across these undocumented choices: FGD can be made to look appreciably more or less human-valid without changing the systems. These effects replicate across two challenge years and three evaluation tiers. We conclude that FGD numbers are not comparable across papers unless the window, seed, scoring space, and feature-extractor training corpus are pinned and reported, and that FGD cannot resolve small differences between closely performing systems."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="gandhi-etal-2026-motions">
<titleInfo>
<title>Same Motions, Different Winners: The Gesture-Generation Leaderboard and Its Human Agreement Are Configuration-Tunable</title>
</titleInfo>
<name type="personal">
<namePart type="given">Deep</namePart>
<namePart type="family">Gandhi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ali</namePart>
<namePart type="family">Asaria</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tony</namePart>
<namePart type="family">Salomone</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Multimodal Interaction in Face-to-Face Dialogue</title>
</titleInfo>
<name type="personal">
<namePart type="given">Esam</namePart>
<namePart type="family">Ghaleb</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ece</namePart>
<namePart type="family">Takmaz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Diego</namePart>
<namePart type="family">Frassinelli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bulat</namePart>
<namePart type="family">Khaertdinov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raquel</namePart>
<namePart type="family">Fernández</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zerrin</namePart>
<namePart type="family">Yumak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Asli</namePart>
<namePart type="family">Ozyurek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-482-8</identifier>
</relatedItem>
<abstract>Fréchet Gesture Distance (FGD) is the standard automatic metric for co-speech gesture generation, the models that synthesize the hand and body motion accompanying speech. Human perceptual studies remain the gold standard, and the GENEA challenges rank systems by them, but such studies are slow and costly, so papers routinely compare systems by FGD instead. FGD is a Fréchet distance in the latent space of a learned pose autoencoder, so it inherits the many choices behind that encoder, most of which papers never report. We ask whether FGD is a reliable proxy for perceptual evaluation under these unreported choices. Holding the systems fixed, we vary the reference tool’s own settings (analysis window, seed, feature versus raw space) and the corpus the feature extractor is trained on, on both tiers of the GENEA 2022 evaluation and on the GENEA 2023 challenge. First, the FGD ranking is not reproducible where it matters: the “best” system is one of two that a per-clip bootstrap shows are statistically tied, so which one wins is a configuration choice, though aggregate rank correlation stays moderate rather than collapsing. Second, and more consequential, FGD’s agreement with human ratings is modest and itself tunable, moving by 0.18 to 0.24 Kendall \ensuremathτ across these undocumented choices: FGD can be made to look appreciably more or less human-valid without changing the systems. These effects replicate across two challenge years and three evaluation tiers. We conclude that FGD numbers are not comparable across papers unless the window, seed, scoring space, and feature-extractor training corpus are pinned and reported, and that FGD cannot resolve small differences between closely performing systems.</abstract>
<identifier type="citekey">gandhi-etal-2026-motions</identifier>
<location>
<url>https://aclanthology.org/2026.mint-1.8/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>72</start>
<end>77</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Same Motions, Different Winners: The Gesture-Generation Leaderboard and Its Human Agreement Are Configuration-Tunable
%A Gandhi, Deep
%A Asaria, Ali
%A Salomone, Tony
%Y Ghaleb, Esam
%Y Takmaz, Ece
%Y Frassinelli, Diego
%Y Khaertdinov, Bulat
%Y Fernández, Raquel
%Y Yumak, Zerrin
%Y Ozyurek, Asli
%S Proceedings of the 1st Workshop on Multimodal Interaction in Face-to-Face Dialogue
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-482-8
%F gandhi-etal-2026-motions
%X Fréchet Gesture Distance (FGD) is the standard automatic metric for co-speech gesture generation, the models that synthesize the hand and body motion accompanying speech. Human perceptual studies remain the gold standard, and the GENEA challenges rank systems by them, but such studies are slow and costly, so papers routinely compare systems by FGD instead. FGD is a Fréchet distance in the latent space of a learned pose autoencoder, so it inherits the many choices behind that encoder, most of which papers never report. We ask whether FGD is a reliable proxy for perceptual evaluation under these unreported choices. Holding the systems fixed, we vary the reference tool’s own settings (analysis window, seed, feature versus raw space) and the corpus the feature extractor is trained on, on both tiers of the GENEA 2022 evaluation and on the GENEA 2023 challenge. First, the FGD ranking is not reproducible where it matters: the “best” system is one of two that a per-clip bootstrap shows are statistically tied, so which one wins is a configuration choice, though aggregate rank correlation stays moderate rather than collapsing. Second, and more consequential, FGD’s agreement with human ratings is modest and itself tunable, moving by 0.18 to 0.24 Kendall \ensuremathτ across these undocumented choices: FGD can be made to look appreciably more or less human-valid without changing the systems. These effects replicate across two challenge years and three evaluation tiers. We conclude that FGD numbers are not comparable across papers unless the window, seed, scoring space, and feature-extractor training corpus are pinned and reported, and that FGD cannot resolve small differences between closely performing systems.
%U https://aclanthology.org/2026.mint-1.8/
%P 72-77
Markdown (Informal)
[Same Motions, Different Winners: The Gesture-Generation Leaderboard and Its Human Agreement Are Configuration-Tunable](https://aclanthology.org/2026.mint-1.8/) (Gandhi et al., MINT 2026)
ACL