@inproceedings{huang-etal-2026-relevance,
title = "Relevance Supervision Can Teach Neural Retrievers to Prefer {LLM}-Generated Text",
author = "Huang, Wei and
Bi, Keping and
Cai, Yinqiong and
Chen, Wei and
Guo, Jiafeng and
Cheng, Xueqi",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2038/",
pages = "43376--43406",
ISBN = "979-8-89176-462-0",
abstract = "Recent studies show that neural retrievers often exhibit source bias, ranking LLM-generated passages above semantically similar human-written ones, which threatens the reliability of modern information access systems. While this bias is often viewed as a model-level property, we show that relevance supervision itself can induce it. Empirically, source bias is clearest in relevance-supervised retrievers, and the same encoders become more LLM-preferring after relevance fine-tuning. We trace this effect to surface-level artifacts in supervised retrieval data: annotated positives and negatives differ systematically in properties such as fluency and lexical specificity, and these differences parallel those between LLM-generated and human-written passages. As a result, retrievers can learn artifact-based shortcuts for relevance. We further show theoretically that, under contrastive learning, artifacts that reliably separate positives from negatives can be encoded into learned relevance scores. Motivated by this mechanism, we propose complementary strategies that reduce source bias during training and at inference: balancing artifact distributions in negative sampling and removing the artifact-aligned embedding component, respectively. Overall, our findings support a data-centric view of source bias: retrievers may prefer LLM-generated text not merely because of inherent model properties, but because relevance supervision teaches them to rely on surface-level artifacts."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="huang-etal-2026-relevance">
<titleInfo>
<title>Relevance Supervision Can Teach Neural Retrievers to Prefer LLM-Generated Text</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keping</namePart>
<namePart type="family">Bi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yinqiong</namePart>
<namePart type="family">Cai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiafeng</namePart>
<namePart type="family">Guo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xueqi</namePart>
<namePart type="family">Cheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Recent studies show that neural retrievers often exhibit source bias, ranking LLM-generated passages above semantically similar human-written ones, which threatens the reliability of modern information access systems. While this bias is often viewed as a model-level property, we show that relevance supervision itself can induce it. Empirically, source bias is clearest in relevance-supervised retrievers, and the same encoders become more LLM-preferring after relevance fine-tuning. We trace this effect to surface-level artifacts in supervised retrieval data: annotated positives and negatives differ systematically in properties such as fluency and lexical specificity, and these differences parallel those between LLM-generated and human-written passages. As a result, retrievers can learn artifact-based shortcuts for relevance. We further show theoretically that, under contrastive learning, artifacts that reliably separate positives from negatives can be encoded into learned relevance scores. Motivated by this mechanism, we propose complementary strategies that reduce source bias during training and at inference: balancing artifact distributions in negative sampling and removing the artifact-aligned embedding component, respectively. Overall, our findings support a data-centric view of source bias: retrievers may prefer LLM-generated text not merely because of inherent model properties, but because relevance supervision teaches them to rely on surface-level artifacts.</abstract>
<identifier type="citekey">huang-etal-2026-relevance</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2038/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>43376</start>
<end>43406</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Relevance Supervision Can Teach Neural Retrievers to Prefer LLM-Generated Text
%A Huang, Wei
%A Bi, Keping
%A Cai, Yinqiong
%A Chen, Wei
%A Guo, Jiafeng
%A Cheng, Xueqi
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F huang-etal-2026-relevance
%X Recent studies show that neural retrievers often exhibit source bias, ranking LLM-generated passages above semantically similar human-written ones, which threatens the reliability of modern information access systems. While this bias is often viewed as a model-level property, we show that relevance supervision itself can induce it. Empirically, source bias is clearest in relevance-supervised retrievers, and the same encoders become more LLM-preferring after relevance fine-tuning. We trace this effect to surface-level artifacts in supervised retrieval data: annotated positives and negatives differ systematically in properties such as fluency and lexical specificity, and these differences parallel those between LLM-generated and human-written passages. As a result, retrievers can learn artifact-based shortcuts for relevance. We further show theoretically that, under contrastive learning, artifacts that reliably separate positives from negatives can be encoded into learned relevance scores. Motivated by this mechanism, we propose complementary strategies that reduce source bias during training and at inference: balancing artifact distributions in negative sampling and removing the artifact-aligned embedding component, respectively. Overall, our findings support a data-centric view of source bias: retrievers may prefer LLM-generated text not merely because of inherent model properties, but because relevance supervision teaches them to rely on surface-level artifacts.
%U https://aclanthology.org/2026.findings-emnlp.2038/
%P 43376-43406
Markdown (Informal)
[Relevance Supervision Can Teach Neural Retrievers to Prefer LLM-Generated Text](https://aclanthology.org/2026.findings-emnlp.2038/) (Huang et al., Findings 2026)
ACL