@inproceedings{konior-etal-2026-riemannian,
title = "{R}iemannian Geometry for Pre-trained Language Model Embeddings",
author = "Konior, Szczepan and
Quemy, Alexandre and
Klocek, Przemys{\l}aw and
Sobieski, Bartlomiej and
Cattan, Gr{\'e}goire",
editor = "Wang, Yimu and
Choi, Yee Man and
Wu, Di and
Ouyang, Siqi and
Azadani, Mozhgan Nasr and
Fung, Yi R. (May)",
booktitle = "Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently ({G}round{LM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.groundlm-1.9/",
pages = "113--131",
ISBN = "979-8-89176-478-1",
abstract = "Understanding the geometric structure of pre-trained language model embeddings matters for interpretability and safety. We ask whether sentence-level classification signal lives in the Riemannian geometry of contextual token embeddings, and probe it by extracting per-token pullback metrics from a learned encoder{'}s analytical Jacobian and aggregating them with the Frechet mean on the symmetric positive definite (SPD) manifold, we call this procedure Riemannian Mean Pooling (RMP). Across three datasets with non-trivial linguistic structure (CoLA, CREAK, RTE), RMP outperforms Euclidean mean pooling, while on FEVER-Symmetric, a benchmark constructed to remove annotation-driven lexical artifacts, the method correctly stays at chance. Ablations show that a randomly initialised encoder combined with Frechet aggregation already beats Euclidean pooling before any training, localising the source of the gain to the geometric aggregation rather than to learned manifold structure. The trained encoder adds a further 1.7 to 2.2 AUC points, consistently across the signal-bearing datasets rather than on any one of them."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="konior-etal-2026-riemannian">
<titleInfo>
<title>Riemannian Geometry for Pre-trained Language Model Embeddings</title>
</titleInfo>
<name type="personal">
<namePart type="given">Szczepan</namePart>
<namePart type="family">Konior</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexandre</namePart>
<namePart type="family">Quemy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Przemysław</namePart>
<namePart type="family">Klocek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bartlomiej</namePart>
<namePart type="family">Sobieski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Grégoire</namePart>
<namePart type="family">Cattan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yee</namePart>
<namePart type="given">Man</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siqi</namePart>
<namePart type="family">Ouyang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mozhgan</namePart>
<namePart type="given">Nasr</namePart>
<namePart type="family">Azadani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="given">R</namePart>
<namePart type="given">(May)</namePart>
<namePart type="family">Fung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-478-1</identifier>
</relatedItem>
<abstract>Understanding the geometric structure of pre-trained language model embeddings matters for interpretability and safety. We ask whether sentence-level classification signal lives in the Riemannian geometry of contextual token embeddings, and probe it by extracting per-token pullback metrics from a learned encoder’s analytical Jacobian and aggregating them with the Frechet mean on the symmetric positive definite (SPD) manifold, we call this procedure Riemannian Mean Pooling (RMP). Across three datasets with non-trivial linguistic structure (CoLA, CREAK, RTE), RMP outperforms Euclidean mean pooling, while on FEVER-Symmetric, a benchmark constructed to remove annotation-driven lexical artifacts, the method correctly stays at chance. Ablations show that a randomly initialised encoder combined with Frechet aggregation already beats Euclidean pooling before any training, localising the source of the gain to the geometric aggregation rather than to learned manifold structure. The trained encoder adds a further 1.7 to 2.2 AUC points, consistently across the signal-bearing datasets rather than on any one of them.</abstract>
<identifier type="citekey">konior-etal-2026-riemannian</identifier>
<location>
<url>https://aclanthology.org/2026.groundlm-1.9/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>113</start>
<end>131</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Riemannian Geometry for Pre-trained Language Model Embeddings
%A Konior, Szczepan
%A Quemy, Alexandre
%A Klocek, Przemysław
%A Sobieski, Bartlomiej
%A Cattan, Grégoire
%Y Wang, Yimu
%Y Choi, Yee Man
%Y Wu, Di
%Y Ouyang, Siqi
%Y Azadani, Mozhgan Nasr
%Y Fung, Yi R. (May)
%S Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-478-1
%F konior-etal-2026-riemannian
%X Understanding the geometric structure of pre-trained language model embeddings matters for interpretability and safety. We ask whether sentence-level classification signal lives in the Riemannian geometry of contextual token embeddings, and probe it by extracting per-token pullback metrics from a learned encoder’s analytical Jacobian and aggregating them with the Frechet mean on the symmetric positive definite (SPD) manifold, we call this procedure Riemannian Mean Pooling (RMP). Across three datasets with non-trivial linguistic structure (CoLA, CREAK, RTE), RMP outperforms Euclidean mean pooling, while on FEVER-Symmetric, a benchmark constructed to remove annotation-driven lexical artifacts, the method correctly stays at chance. Ablations show that a randomly initialised encoder combined with Frechet aggregation already beats Euclidean pooling before any training, localising the source of the gain to the geometric aggregation rather than to learned manifold structure. The trained encoder adds a further 1.7 to 2.2 AUC points, consistently across the signal-bearing datasets rather than on any one of them.
%U https://aclanthology.org/2026.groundlm-1.9/
%P 113-131
Markdown (Informal)
[Riemannian Geometry for Pre-trained Language Model Embeddings](https://aclanthology.org/2026.groundlm-1.9/) (Konior et al., GroundLM 2026)
ACL
- Szczepan Konior, Alexandre Quemy, Przemysław Klocek, Bartlomiej Sobieski, and Grégoire Cattan. 2026. Riemannian Geometry for Pre-trained Language Model Embeddings. In Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026), pages 113–131, Budapest, Hungary. Association for Computational Linguistics.