@inproceedings{seo-jang-2026-retrieval,
title = "When Retrieval Similarity Hurts: A Diagnostic Framework for Document Weighting in Black-Box {RAG}",
author = "Seo, Dongju and
Jang, Beakcheol",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.594/",
pages = "12534--12558",
ISBN = "979-8-89176-462-0",
abstract = "Document weighting is an underspecified bottleneck in black-box retrieval-augmented generation. Training-free systems inherit answer-aggregation weights from retrieval similarity, yet this proxy has not been systematically compared with evidence-based alternatives. We propose a unified two-parameter framework that generalizes REPLUG with an evidence term; it is training-free, black-box, and needs no large language model (LLM) calls beyond voting. Similarity-only ({\ensuremath{\lambda}}=0) and evidence-only ({\ensuremath{\lambda}}{\textrightarrow}{\ensuremath{\infty}}) weighting are its limiting cases, so one interpolation parameter separates the retrieval prior from the evidence signal. We evaluate across 3 LLMs (7B{--}70B), 3 open-domain question-answering datasets (31,861 queries), and 3 off-the-shelf evidence types. Cross-encoder and embedding-stability evidence improve exact match in all 9 LLM{--}dataset cells, whereas natural language inference (NLI) fails as a weighting signal, to our knowledge the first systematic evidence of this failure mode. The similarity prior is largely redundant on NQ-Open and PopQA but helps on TriviaQA, where retriever scores track gold documents better; the pattern replicates under the E5 retriever, with smaller gains concentrated on long-tail PopQA. One default transfers across datasets and scales with minimal loss; calibration depends on signal type, not scale. Practically: prefer cross-encoder evidence, avoid NLI, and use vote mass for selective prediction; weighting itself adds sub-millisecond overhead."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="seo-jang-2026-retrieval">
<titleInfo>
<title>When Retrieval Similarity Hurts: A Diagnostic Framework for Document Weighting in Black-Box RAG</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dongju</namePart>
<namePart type="family">Seo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Beakcheol</namePart>
<namePart type="family">Jang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Document weighting is an underspecified bottleneck in black-box retrieval-augmented generation. Training-free systems inherit answer-aggregation weights from retrieval similarity, yet this proxy has not been systematically compared with evidence-based alternatives. We propose a unified two-parameter framework that generalizes REPLUG with an evidence term; it is training-free, black-box, and needs no large language model (LLM) calls beyond voting. Similarity-only (\ensuremathłambda=0) and evidence-only (\ensuremathłambda→\ensuremathınfty) weighting are its limiting cases, so one interpolation parameter separates the retrieval prior from the evidence signal. We evaluate across 3 LLMs (7B–70B), 3 open-domain question-answering datasets (31,861 queries), and 3 off-the-shelf evidence types. Cross-encoder and embedding-stability evidence improve exact match in all 9 LLM–dataset cells, whereas natural language inference (NLI) fails as a weighting signal, to our knowledge the first systematic evidence of this failure mode. The similarity prior is largely redundant on NQ-Open and PopQA but helps on TriviaQA, where retriever scores track gold documents better; the pattern replicates under the E5 retriever, with smaller gains concentrated on long-tail PopQA. One default transfers across datasets and scales with minimal loss; calibration depends on signal type, not scale. Practically: prefer cross-encoder evidence, avoid NLI, and use vote mass for selective prediction; weighting itself adds sub-millisecond overhead.</abstract>
<identifier type="citekey">seo-jang-2026-retrieval</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.594/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>12534</start>
<end>12558</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Retrieval Similarity Hurts: A Diagnostic Framework for Document Weighting in Black-Box RAG
%A Seo, Dongju
%A Jang, Beakcheol
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F seo-jang-2026-retrieval
%X Document weighting is an underspecified bottleneck in black-box retrieval-augmented generation. Training-free systems inherit answer-aggregation weights from retrieval similarity, yet this proxy has not been systematically compared with evidence-based alternatives. We propose a unified two-parameter framework that generalizes REPLUG with an evidence term; it is training-free, black-box, and needs no large language model (LLM) calls beyond voting. Similarity-only (\ensuremathłambda=0) and evidence-only (\ensuremathłambda→\ensuremathınfty) weighting are its limiting cases, so one interpolation parameter separates the retrieval prior from the evidence signal. We evaluate across 3 LLMs (7B–70B), 3 open-domain question-answering datasets (31,861 queries), and 3 off-the-shelf evidence types. Cross-encoder and embedding-stability evidence improve exact match in all 9 LLM–dataset cells, whereas natural language inference (NLI) fails as a weighting signal, to our knowledge the first systematic evidence of this failure mode. The similarity prior is largely redundant on NQ-Open and PopQA but helps on TriviaQA, where retriever scores track gold documents better; the pattern replicates under the E5 retriever, with smaller gains concentrated on long-tail PopQA. One default transfers across datasets and scales with minimal loss; calibration depends on signal type, not scale. Practically: prefer cross-encoder evidence, avoid NLI, and use vote mass for selective prediction; weighting itself adds sub-millisecond overhead.
%U https://aclanthology.org/2026.findings-emnlp.594/
%P 12534-12558
Markdown (Informal)
[When Retrieval Similarity Hurts: A Diagnostic Framework for Document Weighting in Black-Box RAG](https://aclanthology.org/2026.findings-emnlp.594/) (Seo & Jang, Findings 2026)
ACL