@inproceedings{frank-afli-2026-harder,
title = "The Harder Text Embedding Benchmark ({HTEB}): Beyond One-dimensional Static Robustness",
author = "Frank, Manuel and
Afli, Haithem",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1044/",
pages = "23973--24005",
ISBN = "979-8-89176-459-0",
abstract = "Embedding benchmarks like MTEB report a single score per model, implicitly treating robustness as a static, scalar property. We argue that robustness is multidimensional, since models respond differently to different types of variation, and requires dynamic evaluation to expose failures hidden by static benchmarks. We introduce the Harder Text Embedding Benchmark (HTEB), a dynamic evaluation framework that challenges model robustness along three practically interpretable axes (Lexical/Stylistic, Length and Language) by stochastically transforming inputs at evaluation time with an LLM. Evaluating 16 open-weight embedding models on 32 datasets covering 42 languages under transformations validated by 4,800 individual human ratings on an English subsample, supplemented by a Spanish-source evaluation and an exploratory STS-B study of pair-level label preservation, we find three patterns: (1) Models exhibit partly decoupled robustness profiles across axes. (2) Across three model families, scale increases absolute scores but does not close the gap between original and transformed evaluations. Scaling tends to improve specifically the Language axis. (3) English datasets are more sensitive to HTEB transformations than multilingual datasets. This demonstrates that HTEB identifies strengths and weaknesses of models along deployment-relevant axes, challenging embedding benchmarks and arguing for multidimensional, dynamic robustness evaluation. We make HTEB{'}s code publicly available."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="frank-afli-2026-harder">
<titleInfo>
<title>The Harder Text Embedding Benchmark (HTEB): Beyond One-dimensional Static Robustness</title>
</titleInfo>
<name type="personal">
<namePart type="given">Manuel</namePart>
<namePart type="family">Frank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haithem</namePart>
<namePart type="family">Afli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Embedding benchmarks like MTEB report a single score per model, implicitly treating robustness as a static, scalar property. We argue that robustness is multidimensional, since models respond differently to different types of variation, and requires dynamic evaluation to expose failures hidden by static benchmarks. We introduce the Harder Text Embedding Benchmark (HTEB), a dynamic evaluation framework that challenges model robustness along three practically interpretable axes (Lexical/Stylistic, Length and Language) by stochastically transforming inputs at evaluation time with an LLM. Evaluating 16 open-weight embedding models on 32 datasets covering 42 languages under transformations validated by 4,800 individual human ratings on an English subsample, supplemented by a Spanish-source evaluation and an exploratory STS-B study of pair-level label preservation, we find three patterns: (1) Models exhibit partly decoupled robustness profiles across axes. (2) Across three model families, scale increases absolute scores but does not close the gap between original and transformed evaluations. Scaling tends to improve specifically the Language axis. (3) English datasets are more sensitive to HTEB transformations than multilingual datasets. This demonstrates that HTEB identifies strengths and weaknesses of models along deployment-relevant axes, challenging embedding benchmarks and arguing for multidimensional, dynamic robustness evaluation. We make HTEB’s code publicly available.</abstract>
<identifier type="citekey">frank-afli-2026-harder</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1044/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>23973</start>
<end>24005</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Harder Text Embedding Benchmark (HTEB): Beyond One-dimensional Static Robustness
%A Frank, Manuel
%A Afli, Haithem
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F frank-afli-2026-harder
%X Embedding benchmarks like MTEB report a single score per model, implicitly treating robustness as a static, scalar property. We argue that robustness is multidimensional, since models respond differently to different types of variation, and requires dynamic evaluation to expose failures hidden by static benchmarks. We introduce the Harder Text Embedding Benchmark (HTEB), a dynamic evaluation framework that challenges model robustness along three practically interpretable axes (Lexical/Stylistic, Length and Language) by stochastically transforming inputs at evaluation time with an LLM. Evaluating 16 open-weight embedding models on 32 datasets covering 42 languages under transformations validated by 4,800 individual human ratings on an English subsample, supplemented by a Spanish-source evaluation and an exploratory STS-B study of pair-level label preservation, we find three patterns: (1) Models exhibit partly decoupled robustness profiles across axes. (2) Across three model families, scale increases absolute scores but does not close the gap between original and transformed evaluations. Scaling tends to improve specifically the Language axis. (3) English datasets are more sensitive to HTEB transformations than multilingual datasets. This demonstrates that HTEB identifies strengths and weaknesses of models along deployment-relevant axes, challenging embedding benchmarks and arguing for multidimensional, dynamic robustness evaluation. We make HTEB’s code publicly available.
%U https://aclanthology.org/2026.emnlp-main.1044/
%P 23973-24005
Markdown (Informal)
[The Harder Text Embedding Benchmark (HTEB): Beyond One-dimensional Static Robustness](https://aclanthology.org/2026.emnlp-main.1044/) (Frank & Afli, EMNLP 2026)
ACL