@inproceedings{keuren-etal-2026-finding,
title = "Finding Meaning in Embeddings: Concept Separation Curves",
author = "Keuren, Paul and
Ponsen, Marc and
Bagheri, Robert Ayoub",
editor = "Zhao, Jin and
Post, Claire Benet and
Hoefer, Elizabeth",
booktitle = "Proceedings of The Seventh International Workshop on Designing Meaning Representations ({DMR} 2026) @ {LREC} 2026",
month = may,
year = "2026",
address = "Palma, Mallorca (Spain)",
publisher = "ELRA Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.dmr-1.8/",
doi = "10.63317/2vr5jdznujwd",
pages = "91--101",
abstract = "Sentence embedding techniques aim to encode key concepts of a sentence{'}s meaning in a vector space. However, the majority of evaluation approaches for sentence embedding quality rely on the use of additional classifiers or downstream tasks. These additional components make it unclear whether good results stem from the embedding itself or from the classifier{'}s behaviour. In this paper, we propose a novel method for evaluating the effectiveness of sentence embedding methods in capturing sentence-level concepts. Our approach is classifier-independent, allowing for an objective assessment of the model{'}s performance. The approach adopted in this study involves the systematic introduction of syntactic noise and semantic negations into sentences, with the subsequent quantification of their relative effects on the resulting embeddings. The visualisation of these effects is facilitated by Concept Separation Curves, which show the model{'}s capacity to differentiate between conceptual and surface-level variations. By leveraging data from multiple domains, employing both Dutch and English languages, and examining sentence lengths, this study offers a compelling demonstration that Concept Separation Curves provide an interpretable, reproducible, and cross-model approach for evaluating the conceptual stability of sentence embeddings. The open-source code and a live interactive demo are available upon acceptance."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="keuren-etal-2026-finding">
<titleInfo>
<title>Finding Meaning in Embeddings: Concept Separation Curves</title>
</titleInfo>
<name type="personal">
<namePart type="given">Paul</namePart>
<namePart type="family">Keuren</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marc</namePart>
<namePart type="family">Ponsen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Robert</namePart>
<namePart type="given">Ayoub</namePart>
<namePart type="family">Bagheri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of The Seventh International Workshop on Designing Meaning Representations (DMR 2026) @ LREC 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jin</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Claire</namePart>
<namePart type="given">Benet</namePart>
<namePart type="family">Post</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elizabeth</namePart>
<namePart type="family">Hoefer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma, Mallorca (Spain)</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Sentence embedding techniques aim to encode key concepts of a sentence’s meaning in a vector space. However, the majority of evaluation approaches for sentence embedding quality rely on the use of additional classifiers or downstream tasks. These additional components make it unclear whether good results stem from the embedding itself or from the classifier’s behaviour. In this paper, we propose a novel method for evaluating the effectiveness of sentence embedding methods in capturing sentence-level concepts. Our approach is classifier-independent, allowing for an objective assessment of the model’s performance. The approach adopted in this study involves the systematic introduction of syntactic noise and semantic negations into sentences, with the subsequent quantification of their relative effects on the resulting embeddings. The visualisation of these effects is facilitated by Concept Separation Curves, which show the model’s capacity to differentiate between conceptual and surface-level variations. By leveraging data from multiple domains, employing both Dutch and English languages, and examining sentence lengths, this study offers a compelling demonstration that Concept Separation Curves provide an interpretable, reproducible, and cross-model approach for evaluating the conceptual stability of sentence embeddings. The open-source code and a live interactive demo are available upon acceptance.</abstract>
<identifier type="citekey">keuren-etal-2026-finding</identifier>
<identifier type="doi">10.63317/2vr5jdznujwd</identifier>
<location>
<url>https://aclanthology.org/2026.dmr-1.8/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>91</start>
<end>101</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Finding Meaning in Embeddings: Concept Separation Curves
%A Keuren, Paul
%A Ponsen, Marc
%A Bagheri, Robert Ayoub
%Y Zhao, Jin
%Y Post, Claire Benet
%Y Hoefer, Elizabeth
%S Proceedings of The Seventh International Workshop on Designing Meaning Representations (DMR 2026) @ LREC 2026
%D 2026
%8 May
%I ELRA Language Resources Association (ELRA)
%C Palma, Mallorca (Spain)
%F keuren-etal-2026-finding
%X Sentence embedding techniques aim to encode key concepts of a sentence’s meaning in a vector space. However, the majority of evaluation approaches for sentence embedding quality rely on the use of additional classifiers or downstream tasks. These additional components make it unclear whether good results stem from the embedding itself or from the classifier’s behaviour. In this paper, we propose a novel method for evaluating the effectiveness of sentence embedding methods in capturing sentence-level concepts. Our approach is classifier-independent, allowing for an objective assessment of the model’s performance. The approach adopted in this study involves the systematic introduction of syntactic noise and semantic negations into sentences, with the subsequent quantification of their relative effects on the resulting embeddings. The visualisation of these effects is facilitated by Concept Separation Curves, which show the model’s capacity to differentiate between conceptual and surface-level variations. By leveraging data from multiple domains, employing both Dutch and English languages, and examining sentence lengths, this study offers a compelling demonstration that Concept Separation Curves provide an interpretable, reproducible, and cross-model approach for evaluating the conceptual stability of sentence embeddings. The open-source code and a live interactive demo are available upon acceptance.
%R 10.63317/2vr5jdznujwd
%U https://aclanthology.org/2026.dmr-1.8/
%U https://doi.org/10.63317/2vr5jdznujwd
%P 91-101
Markdown (Informal)
[Finding Meaning in Embeddings: Concept Separation Curves](https://aclanthology.org/2026.dmr-1.8/) (Keuren et al., DMR 2026)
ACL
- Paul Keuren, Marc Ponsen, and Robert Ayoub Bagheri. 2026. Finding Meaning in Embeddings: Concept Separation Curves. In Proceedings of The Seventh International Workshop on Designing Meaning Representations (DMR 2026) @ LREC 2026, pages 91–101, Palma, Mallorca (Spain). ELRA Language Resources Association (ELRA).