@inproceedings{wilinski-farber-2026-sciembed,
title = "{S}ci{E}mbed: Citation-Context Supervision for Scientific Document Embeddings",
author = {Wilinski, Jonas and
F{\"a}rber, Michael},
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2057/",
pages = "43793--43819",
ISBN = "979-8-89176-462-0",
abstract = "Open scientific document embedders are typically trained on binary citation edges between paper title-abstract pairs. This signal discards the citation context sentence: a domain-expert description of the cited paper, written by the citing author. We use these sentences as the primary contrastive supervision signal to train SciEmbed, an open 149M ModernBERT-base encoder. On the official SciRepEval benchmark it reaches 66.85 over three seeds, surpassing prior open scientific embedders such as SPECTER2 and SciNCL. To isolate what the supervision source contributes, we compare against IBM Granite R2, a same-size ModernBERT peer trained on general retrieval data. Using only open citation-context data, SciEmbed matches Granite R2 to within 0.6 on the SciRepEval macro overall while specialising in the opposite direction: it leads regression tasks tied to paper impact by 2.3 points, where Granite R2 instead leads classification, proximity, and search. A 28-configuration sweep over recipes and alternative supervision signals does not improve on this result, consistent with the specialisation tracking the supervision source rather than the training recipe. Training at longer contexts yields monotonic gains on a body-fact retrieval probe within our recipe. We release the model, training pipeline, and evaluation harness as a fully open, citation-context-supervised baseline with a distinct, reproducible specialisation pattern."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wilinski-farber-2026-sciembed">
<titleInfo>
<title>SciEmbed: Citation-Context Supervision for Scientific Document Embeddings</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jonas</namePart>
<namePart type="family">Wilinski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Michael</namePart>
<namePart type="family">Färber</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Open scientific document embedders are typically trained on binary citation edges between paper title-abstract pairs. This signal discards the citation context sentence: a domain-expert description of the cited paper, written by the citing author. We use these sentences as the primary contrastive supervision signal to train SciEmbed, an open 149M ModernBERT-base encoder. On the official SciRepEval benchmark it reaches 66.85 over three seeds, surpassing prior open scientific embedders such as SPECTER2 and SciNCL. To isolate what the supervision source contributes, we compare against IBM Granite R2, a same-size ModernBERT peer trained on general retrieval data. Using only open citation-context data, SciEmbed matches Granite R2 to within 0.6 on the SciRepEval macro overall while specialising in the opposite direction: it leads regression tasks tied to paper impact by 2.3 points, where Granite R2 instead leads classification, proximity, and search. A 28-configuration sweep over recipes and alternative supervision signals does not improve on this result, consistent with the specialisation tracking the supervision source rather than the training recipe. Training at longer contexts yields monotonic gains on a body-fact retrieval probe within our recipe. We release the model, training pipeline, and evaluation harness as a fully open, citation-context-supervised baseline with a distinct, reproducible specialisation pattern.</abstract>
<identifier type="citekey">wilinski-farber-2026-sciembed</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2057/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>43793</start>
<end>43819</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SciEmbed: Citation-Context Supervision for Scientific Document Embeddings
%A Wilinski, Jonas
%A Färber, Michael
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F wilinski-farber-2026-sciembed
%X Open scientific document embedders are typically trained on binary citation edges between paper title-abstract pairs. This signal discards the citation context sentence: a domain-expert description of the cited paper, written by the citing author. We use these sentences as the primary contrastive supervision signal to train SciEmbed, an open 149M ModernBERT-base encoder. On the official SciRepEval benchmark it reaches 66.85 over three seeds, surpassing prior open scientific embedders such as SPECTER2 and SciNCL. To isolate what the supervision source contributes, we compare against IBM Granite R2, a same-size ModernBERT peer trained on general retrieval data. Using only open citation-context data, SciEmbed matches Granite R2 to within 0.6 on the SciRepEval macro overall while specialising in the opposite direction: it leads regression tasks tied to paper impact by 2.3 points, where Granite R2 instead leads classification, proximity, and search. A 28-configuration sweep over recipes and alternative supervision signals does not improve on this result, consistent with the specialisation tracking the supervision source rather than the training recipe. Training at longer contexts yields monotonic gains on a body-fact retrieval probe within our recipe. We release the model, training pipeline, and evaluation harness as a fully open, citation-context-supervised baseline with a distinct, reproducible specialisation pattern.
%U https://aclanthology.org/2026.findings-emnlp.2057/
%P 43793-43819
Markdown (Informal)
[SciEmbed: Citation-Context Supervision for Scientific Document Embeddings](https://aclanthology.org/2026.findings-emnlp.2057/) (Wilinski & Färber, Findings 2026)
ACL