@inproceedings{mahajan-etal-2026-beyond,
title = "Beyond Downstream Task Performances: A Set-Theoretic Evaluation of Sentence Embeddings",
author = "Mahajan, Yash and
Bansal, Naman and
Kader, Faria Binte and
Karmaker, Santu",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.97/",
pages = "2125--2158",
ISBN = "979-8-89176-439-2",
abstract = "Sentence encoders are foundational to a wide range of NLP applications, yet they are predominantly evaluated through performance on downstream tasks. Such task-specific evaluations provide limited insight into whether the resulting sentence representations capture fundamental compositional relationships in a task-independent manner. To address this gap, we introduce a set-theoretic framework for evaluating the compositional properties of sentence embeddings. Drawing on classical set theory, we define six evaluation criteria based on three core set-like text operations: TextOverlap, TextDifference, and TextUnion. We use these criteria to systematically evaluate 19 sentence encoders, including 7 classical models and 12 Large Language Model (LLM)-based encoders. Our results reveal substantial differences in their compositional behavior: notably, SBERT exhibits the strongest overall alignment with the proposed set-theoretic criteria, outperforming even recent LLM-based encoders. We further examine whether these intrinsic compositional properties are associated with downstream performance across five NLP tasks. Among them, summarization exhibits the strongest correlation with the TextOverlap and TextDifference criteria, suggesting that our framework may also provide useful signals about downstream capability without relying exclusively on task-specific evaluation. Finally, we release a benchmark comprising {\textasciitilde}192K samples to support systematic evaluation and future research on the compositional properties of sentence embeddings."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mahajan-etal-2026-beyond">
<titleInfo>
<title>Beyond Downstream Task Performances: A Set-Theoretic Evaluation of Sentence Embeddings</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yash</namePart>
<namePart type="family">Mahajan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Naman</namePart>
<namePart type="family">Bansal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Faria</namePart>
<namePart type="given">Binte</namePart>
<namePart type="family">Kader</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Santu</namePart>
<namePart type="family">Karmaker</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Sentence encoders are foundational to a wide range of NLP applications, yet they are predominantly evaluated through performance on downstream tasks. Such task-specific evaluations provide limited insight into whether the resulting sentence representations capture fundamental compositional relationships in a task-independent manner. To address this gap, we introduce a set-theoretic framework for evaluating the compositional properties of sentence embeddings. Drawing on classical set theory, we define six evaluation criteria based on three core set-like text operations: TextOverlap, TextDifference, and TextUnion. We use these criteria to systematically evaluate 19 sentence encoders, including 7 classical models and 12 Large Language Model (LLM)-based encoders. Our results reveal substantial differences in their compositional behavior: notably, SBERT exhibits the strongest overall alignment with the proposed set-theoretic criteria, outperforming even recent LLM-based encoders. We further examine whether these intrinsic compositional properties are associated with downstream performance across five NLP tasks. Among them, summarization exhibits the strongest correlation with the TextOverlap and TextDifference criteria, suggesting that our framework may also provide useful signals about downstream capability without relying exclusively on task-specific evaluation. Finally, we release a benchmark comprising ~192K samples to support systematic evaluation and future research on the compositional properties of sentence embeddings.</abstract>
<identifier type="citekey">mahajan-etal-2026-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.97/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>2125</start>
<end>2158</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Downstream Task Performances: A Set-Theoretic Evaluation of Sentence Embeddings
%A Mahajan, Yash
%A Bansal, Naman
%A Kader, Faria Binte
%A Karmaker, Santu
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F mahajan-etal-2026-beyond
%X Sentence encoders are foundational to a wide range of NLP applications, yet they are predominantly evaluated through performance on downstream tasks. Such task-specific evaluations provide limited insight into whether the resulting sentence representations capture fundamental compositional relationships in a task-independent manner. To address this gap, we introduce a set-theoretic framework for evaluating the compositional properties of sentence embeddings. Drawing on classical set theory, we define six evaluation criteria based on three core set-like text operations: TextOverlap, TextDifference, and TextUnion. We use these criteria to systematically evaluate 19 sentence encoders, including 7 classical models and 12 Large Language Model (LLM)-based encoders. Our results reveal substantial differences in their compositional behavior: notably, SBERT exhibits the strongest overall alignment with the proposed set-theoretic criteria, outperforming even recent LLM-based encoders. We further examine whether these intrinsic compositional properties are associated with downstream performance across five NLP tasks. Among them, summarization exhibits the strongest correlation with the TextOverlap and TextDifference criteria, suggesting that our framework may also provide useful signals about downstream capability without relying exclusively on task-specific evaluation. Finally, we release a benchmark comprising ~192K samples to support systematic evaluation and future research on the compositional properties of sentence embeddings.
%U https://aclanthology.org/2026.aacl-long.97/
%P 2125-2158
Markdown (Informal)
[Beyond Downstream Task Performances: A Set-Theoretic Evaluation of Sentence Embeddings](https://aclanthology.org/2026.aacl-long.97/) (Mahajan et al., AACL-IJCNLP 2026)
ACL
- Yash Mahajan, Naman Bansal, Faria Binte Kader, and Santu Karmaker. 2026. Beyond Downstream Task Performances: A Set-Theoretic Evaluation of Sentence Embeddings. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 2125–2158, Hengqin, China. Association for Computational Linguistics.