@inproceedings{yamagiwa-etal-2023-discovering,
title = "Discovering Universal Geometry in Embeddings with {ICA}",
author = "Yamagiwa, Hiroaki and
Oyama, Momose and
Shimodaira, Hidetoshi",
editor = "Bouamor, Houda and
Pino, Juan and
Bali, Kalika",
booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing",
month = dec,
year = "2023",
address = "Singapore",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2023.emnlp-main.283",
doi = "10.18653/v1/2023.emnlp-main.283",
pages = "4647--4675",
abstract = "This study utilizes Independent Component Analysis (ICA) to unveil a consistent semantic structure within embeddings of words or images. Our approach extracts independent semantic components from the embeddings of a pre-trained model by leveraging anisotropic information that remains after the whitening process in Principal Component Analysis (PCA). We demonstrate that each embedding can be expressed as a composition of a few intrinsic interpretable axes and that these semantic axes remain consistent across different languages, algorithms, and modalities. The discovery of a universal semantic structure in the geometric patterns of embeddings enhances our understanding of the representations in embeddings.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yamagiwa-etal-2023-discovering">
<titleInfo>
<title>Discovering Universal Geometry in Embeddings with ICA</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hiroaki</namePart>
<namePart type="family">Yamagiwa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Momose</namePart>
<namePart type="family">Oyama</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hidetoshi</namePart>
<namePart type="family">Shimodaira</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2023-12</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Houda</namePart>
<namePart type="family">Bouamor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Juan</namePart>
<namePart type="family">Pino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kalika</namePart>
<namePart type="family">Bali</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Singapore</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>This study utilizes Independent Component Analysis (ICA) to unveil a consistent semantic structure within embeddings of words or images. Our approach extracts independent semantic components from the embeddings of a pre-trained model by leveraging anisotropic information that remains after the whitening process in Principal Component Analysis (PCA). We demonstrate that each embedding can be expressed as a composition of a few intrinsic interpretable axes and that these semantic axes remain consistent across different languages, algorithms, and modalities. The discovery of a universal semantic structure in the geometric patterns of embeddings enhances our understanding of the representations in embeddings.</abstract>
<identifier type="citekey">yamagiwa-etal-2023-discovering</identifier>
<identifier type="doi">10.18653/v1/2023.emnlp-main.283</identifier>
<location>
<url>https://aclanthology.org/2023.emnlp-main.283</url>
</location>
<part>
<date>2023-12</date>
<extent unit="page">
<start>4647</start>
<end>4675</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Discovering Universal Geometry in Embeddings with ICA
%A Yamagiwa, Hiroaki
%A Oyama, Momose
%A Shimodaira, Hidetoshi
%Y Bouamor, Houda
%Y Pino, Juan
%Y Bali, Kalika
%S Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing
%D 2023
%8 December
%I Association for Computational Linguistics
%C Singapore
%F yamagiwa-etal-2023-discovering
%X This study utilizes Independent Component Analysis (ICA) to unveil a consistent semantic structure within embeddings of words or images. Our approach extracts independent semantic components from the embeddings of a pre-trained model by leveraging anisotropic information that remains after the whitening process in Principal Component Analysis (PCA). We demonstrate that each embedding can be expressed as a composition of a few intrinsic interpretable axes and that these semantic axes remain consistent across different languages, algorithms, and modalities. The discovery of a universal semantic structure in the geometric patterns of embeddings enhances our understanding of the representations in embeddings.
%R 10.18653/v1/2023.emnlp-main.283
%U https://aclanthology.org/2023.emnlp-main.283
%U https://doi.org/10.18653/v1/2023.emnlp-main.283
%P 4647-4675
Markdown (Informal)
[Discovering Universal Geometry in Embeddings with ICA](https://aclanthology.org/2023.emnlp-main.283) (Yamagiwa et al., EMNLP 2023)
ACL
- Hiroaki Yamagiwa, Momose Oyama, and Hidetoshi Shimodaira. 2023. Discovering Universal Geometry in Embeddings with ICA. In Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, pages 4647–4675, Singapore. Association for Computational Linguistics.