@inproceedings{li-etal-2026-geometry-first,
title = "Geometry First: Shaping Sentence Embeddings with Controlled Singular Value Regularization",
author = "Li, Senyu and
Yu, Hao and
Wang, Ziyuan and
Adelani, David Ifeoluwa",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.682/",
pages = "15711--15726",
ISBN = "979-8-89176-459-0",
abstract = "Contrastive learning is widely used for sentence embedding training, yet it provides limited control over the global geometry of the learned embedding space. Sentence embeddings often exhibit spectral concentration, where a small set of directions dominates the representation while many others remain weakly active. We present singular value regularization (SVR), a batch-level framework for shaping sentence embedding spaces by directly controlling the singular value spectrum during training. SVR enables explicit control over both the spectral floor and the number of regularized directions. Experiments show consistent improvements across encoder backbones, pooling strategies, English evaluation tasks, and multilingual settings. Our analysis further shows that the best regularization range is not full-dimensional flattening, but a controlled active-dimensional region aligned with the naturally active subspace of pretrained representations. Moreover, the strongest results are obtained with a large SVR weight, indicating that spectral shaping can provide a substantial optimization signal alongside contrastive supervision. These findings motivate a geometry-aware perspective on sentence embedding learning, in which semantic supervision and explicit control of the embedding-space geometry play complementary roles."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-etal-2026-geometry-first">
<titleInfo>
<title>Geometry First: Shaping Sentence Embeddings with Controlled Singular Value Regularization</title>
</titleInfo>
<name type="personal">
<namePart type="given">Senyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hao</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziyuan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Ifeoluwa</namePart>
<namePart type="family">Adelani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Contrastive learning is widely used for sentence embedding training, yet it provides limited control over the global geometry of the learned embedding space. Sentence embeddings often exhibit spectral concentration, where a small set of directions dominates the representation while many others remain weakly active. We present singular value regularization (SVR), a batch-level framework for shaping sentence embedding spaces by directly controlling the singular value spectrum during training. SVR enables explicit control over both the spectral floor and the number of regularized directions. Experiments show consistent improvements across encoder backbones, pooling strategies, English evaluation tasks, and multilingual settings. Our analysis further shows that the best regularization range is not full-dimensional flattening, but a controlled active-dimensional region aligned with the naturally active subspace of pretrained representations. Moreover, the strongest results are obtained with a large SVR weight, indicating that spectral shaping can provide a substantial optimization signal alongside contrastive supervision. These findings motivate a geometry-aware perspective on sentence embedding learning, in which semantic supervision and explicit control of the embedding-space geometry play complementary roles.</abstract>
<identifier type="citekey">li-etal-2026-geometry-first</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.682/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>15711</start>
<end>15726</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Geometry First: Shaping Sentence Embeddings with Controlled Singular Value Regularization
%A Li, Senyu
%A Yu, Hao
%A Wang, Ziyuan
%A Adelani, David Ifeoluwa
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F li-etal-2026-geometry-first
%X Contrastive learning is widely used for sentence embedding training, yet it provides limited control over the global geometry of the learned embedding space. Sentence embeddings often exhibit spectral concentration, where a small set of directions dominates the representation while many others remain weakly active. We present singular value regularization (SVR), a batch-level framework for shaping sentence embedding spaces by directly controlling the singular value spectrum during training. SVR enables explicit control over both the spectral floor and the number of regularized directions. Experiments show consistent improvements across encoder backbones, pooling strategies, English evaluation tasks, and multilingual settings. Our analysis further shows that the best regularization range is not full-dimensional flattening, but a controlled active-dimensional region aligned with the naturally active subspace of pretrained representations. Moreover, the strongest results are obtained with a large SVR weight, indicating that spectral shaping can provide a substantial optimization signal alongside contrastive supervision. These findings motivate a geometry-aware perspective on sentence embedding learning, in which semantic supervision and explicit control of the embedding-space geometry play complementary roles.
%U https://aclanthology.org/2026.emnlp-main.682/
%P 15711-15726
Markdown (Informal)
[Geometry First: Shaping Sentence Embeddings with Controlled Singular Value Regularization](https://aclanthology.org/2026.emnlp-main.682/) (Li et al., EMNLP 2026)
ACL