@inproceedings{morita-etal-2026-longitudinal,
title = "Longitudinal-aware Evaluation of Retrieval-Augmented Generation over Evolving Credit Policy Documents",
author = "Morita, Kyosuke and
Sanderson, Jack and
Hewitt, Holly",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.40/",
pages = "667--680",
ISBN = "979-8-89176-460-6",
abstract = "We present a retrieval-augmented generation (RAG) system in production since April 2026 for credit policy question answering, a setting in which the corpus is revised several times per quarter: a single revision changes thresholds, narrows scopes, and restructures tables. Static test sets silently decay under this regime, and scores computed against successive corpus versions cannot separate system drift from corpus drift. We propose a longitudinal-aware evaluation framework that makes RAG quality over an evolving corpus both measurable and attributable. It combines (i) a change-type taxonomy {---} Stable, Evolved, Added, Removed {---} built on a deterministic, model-independent alignment of document subsections keyed to how each gold-supporting provision changed between versions, making explicit which scores remain comparable across releases; (ii) a versioned, class-stratified synthetic QA pipeline that keeps the test set current at revision cadence; and (iii) a statement-level LLM judge for compound policy answers; each component is calibrated against subject-matter experts. Across three corpus versions, a conventional longitudinal-unaware protocol would report a recall decline of up to 25.7 points. However, paired comparisons on answer-invariant items show no evidence of regression in the unchanged system."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="morita-etal-2026-longitudinal">
<titleInfo>
<title>Longitudinal-aware Evaluation of Retrieval-Augmented Generation over Evolving Credit Policy Documents</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kyosuke</namePart>
<namePart type="family">Morita</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jack</namePart>
<namePart type="family">Sanderson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Holly</namePart>
<namePart type="family">Hewitt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>We present a retrieval-augmented generation (RAG) system in production since April 2026 for credit policy question answering, a setting in which the corpus is revised several times per quarter: a single revision changes thresholds, narrows scopes, and restructures tables. Static test sets silently decay under this regime, and scores computed against successive corpus versions cannot separate system drift from corpus drift. We propose a longitudinal-aware evaluation framework that makes RAG quality over an evolving corpus both measurable and attributable. It combines (i) a change-type taxonomy — Stable, Evolved, Added, Removed — built on a deterministic, model-independent alignment of document subsections keyed to how each gold-supporting provision changed between versions, making explicit which scores remain comparable across releases; (ii) a versioned, class-stratified synthetic QA pipeline that keeps the test set current at revision cadence; and (iii) a statement-level LLM judge for compound policy answers; each component is calibrated against subject-matter experts. Across three corpus versions, a conventional longitudinal-unaware protocol would report a recall decline of up to 25.7 points. However, paired comparisons on answer-invariant items show no evidence of regression in the unchanged system.</abstract>
<identifier type="citekey">morita-etal-2026-longitudinal</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.40/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>667</start>
<end>680</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Longitudinal-aware Evaluation of Retrieval-Augmented Generation over Evolving Credit Policy Documents
%A Morita, Kyosuke
%A Sanderson, Jack
%A Hewitt, Holly
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F morita-etal-2026-longitudinal
%X We present a retrieval-augmented generation (RAG) system in production since April 2026 for credit policy question answering, a setting in which the corpus is revised several times per quarter: a single revision changes thresholds, narrows scopes, and restructures tables. Static test sets silently decay under this regime, and scores computed against successive corpus versions cannot separate system drift from corpus drift. We propose a longitudinal-aware evaluation framework that makes RAG quality over an evolving corpus both measurable and attributable. It combines (i) a change-type taxonomy — Stable, Evolved, Added, Removed — built on a deterministic, model-independent alignment of document subsections keyed to how each gold-supporting provision changed between versions, making explicit which scores remain comparable across releases; (ii) a versioned, class-stratified synthetic QA pipeline that keeps the test set current at revision cadence; and (iii) a statement-level LLM judge for compound policy answers; each component is calibrated against subject-matter experts. Across three corpus versions, a conventional longitudinal-unaware protocol would report a recall decline of up to 25.7 points. However, paired comparisons on answer-invariant items show no evidence of regression in the unchanged system.
%U https://aclanthology.org/2026.emnlp-industry.40/
%P 667-680
Markdown (Informal)
[Longitudinal-aware Evaluation of Retrieval-Augmented Generation over Evolving Credit Policy Documents](https://aclanthology.org/2026.emnlp-industry.40/) (Morita et al., EMNLP 2026)
ACL