@inproceedings{baklanov-etal-2026-countermem,
title = "{C}ounter{M}em: Counterfactual Evaluation of Non-Parametric Memory Use in Retrieval-Augmented Generation",
author = "Baklanov, Mikhail and
Barkan, Oren and
Koenigstein, Noam",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1054/",
pages = "22332--22348",
ISBN = "979-8-89176-462-0",
abstract = "Retrieval-augmented generation (RAG) is usually evaluated from one observed trace: the query, the retrieved context, and the generated answer. Such evaluations can tell us whether the answer is consistent with the retrieved context, but not whether the system would still give the same answer if that context were removed, updated, or contradicted. We introduce COUNTERMEM, an evaluation protocol for RAG systems that are expected to answer from retrieved evidence: it keeps the RAG configuration fixed, changes only the retrieved context, regenerates the answer, and records whether the answer changes in the expected direction. Across RAGBench, ExpertQA, and DelucionQA, COUNTERMEM complements observed-trace faithfulness by exposing failures it is not designed to see. Configurations that look equivalent under RAGAS faithfulness behave very differently once their evidence is removed, updated, or contradicted, and among RAGChecker-approved answers more than half silently miss coherent contradictions and over a third ignore coherent updates. The failures persist across generator and judge families, the behavioral detectors agree with blind human annotators as closely as the annotators agree with each other, and row-level intervention traces localize failures to retrieval, reranking, or generation. Artifact (protocol, prompts, detectors, scripts, audit traces): https://github.com/DeltaLabTLV/CounterMem."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="baklanov-etal-2026-countermem">
<titleInfo>
<title>CounterMem: Counterfactual Evaluation of Non-Parametric Memory Use in Retrieval-Augmented Generation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mikhail</namePart>
<namePart type="family">Baklanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Oren</namePart>
<namePart type="family">Barkan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Noam</namePart>
<namePart type="family">Koenigstein</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Retrieval-augmented generation (RAG) is usually evaluated from one observed trace: the query, the retrieved context, and the generated answer. Such evaluations can tell us whether the answer is consistent with the retrieved context, but not whether the system would still give the same answer if that context were removed, updated, or contradicted. We introduce COUNTERMEM, an evaluation protocol for RAG systems that are expected to answer from retrieved evidence: it keeps the RAG configuration fixed, changes only the retrieved context, regenerates the answer, and records whether the answer changes in the expected direction. Across RAGBench, ExpertQA, and DelucionQA, COUNTERMEM complements observed-trace faithfulness by exposing failures it is not designed to see. Configurations that look equivalent under RAGAS faithfulness behave very differently once their evidence is removed, updated, or contradicted, and among RAGChecker-approved answers more than half silently miss coherent contradictions and over a third ignore coherent updates. The failures persist across generator and judge families, the behavioral detectors agree with blind human annotators as closely as the annotators agree with each other, and row-level intervention traces localize failures to retrieval, reranking, or generation. Artifact (protocol, prompts, detectors, scripts, audit traces): https://github.com/DeltaLabTLV/CounterMem.</abstract>
<identifier type="citekey">baklanov-etal-2026-countermem</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1054/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>22332</start>
<end>22348</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T CounterMem: Counterfactual Evaluation of Non-Parametric Memory Use in Retrieval-Augmented Generation
%A Baklanov, Mikhail
%A Barkan, Oren
%A Koenigstein, Noam
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F baklanov-etal-2026-countermem
%X Retrieval-augmented generation (RAG) is usually evaluated from one observed trace: the query, the retrieved context, and the generated answer. Such evaluations can tell us whether the answer is consistent with the retrieved context, but not whether the system would still give the same answer if that context were removed, updated, or contradicted. We introduce COUNTERMEM, an evaluation protocol for RAG systems that are expected to answer from retrieved evidence: it keeps the RAG configuration fixed, changes only the retrieved context, regenerates the answer, and records whether the answer changes in the expected direction. Across RAGBench, ExpertQA, and DelucionQA, COUNTERMEM complements observed-trace faithfulness by exposing failures it is not designed to see. Configurations that look equivalent under RAGAS faithfulness behave very differently once their evidence is removed, updated, or contradicted, and among RAGChecker-approved answers more than half silently miss coherent contradictions and over a third ignore coherent updates. The failures persist across generator and judge families, the behavioral detectors agree with blind human annotators as closely as the annotators agree with each other, and row-level intervention traces localize failures to retrieval, reranking, or generation. Artifact (protocol, prompts, detectors, scripts, audit traces): https://github.com/DeltaLabTLV/CounterMem.
%U https://aclanthology.org/2026.findings-emnlp.1054/
%P 22332-22348
Markdown (Informal)
[CounterMem: Counterfactual Evaluation of Non-Parametric Memory Use in Retrieval-Augmented Generation](https://aclanthology.org/2026.findings-emnlp.1054/) (Baklanov et al., Findings 2026)
ACL