@inproceedings{morita-2026-verge,
title = "{VERGE}: Verification-Enhanced Generation of Multi-Hop Evaluation Datasets for Task-Specific {RAG}",
author = "Morita, Kyosuke",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.139/",
pages = "3036--3062",
ISBN = "979-8-89176-439-2",
abstract = "Evaluating retrieval-augmented generation (RAG) on custom knowledge bases requires task-specific benchmarks of multi-chunk evidence synthesis, yet most existing generation pipelines apply quality control only after the full question set has been solved. We present VERGE, which moves quality control inside the generation loop: a verification agent targets, per item and without an external solver pass, an operational multi-chunk criterion and a QA-integrity criterion, refining items that fail and discarding those it cannot repair. From 11,052 candidates across six domains, VERGE yields 2,854 verified multiple-choice items targeting aggregation- and comparison-style synthesis over topically coherent evidence. We contribute a method-agnostic validation protocol {---} exhaustive strict-subset ablation, chunk-removal ablation, format-unconfounded open-ended re-evaluation, and inter-chunk-similarity stratification {---} and use it to localise where the multi-chunk dependence holds. On a stratified sample (n = 250), 69.8{\%} of items not answerable closed-book cannot be answered from any strict subset of their supporting chunks (strict-necessity violation rate 15.2{\%}). A human annotation study finds the verifier reliable (Cohen{'}s $\kappa$ = 0.903) and VERGE items higher in QA integrity and distractor quality than the closest prior method (p $<$ 0.001)."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="morita-2026-verge">
<titleInfo>
<title>VERGE: Verification-Enhanced Generation of Multi-Hop Evaluation Datasets for Task-Specific RAG</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kyosuke</namePart>
<namePart type="family">Morita</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Evaluating retrieval-augmented generation (RAG) on custom knowledge bases requires task-specific benchmarks of multi-chunk evidence synthesis, yet most existing generation pipelines apply quality control only after the full question set has been solved. We present VERGE, which moves quality control inside the generation loop: a verification agent targets, per item and without an external solver pass, an operational multi-chunk criterion and a QA-integrity criterion, refining items that fail and discarding those it cannot repair. From 11,052 candidates across six domains, VERGE yields 2,854 verified multiple-choice items targeting aggregation- and comparison-style synthesis over topically coherent evidence. We contribute a method-agnostic validation protocol — exhaustive strict-subset ablation, chunk-removal ablation, format-unconfounded open-ended re-evaluation, and inter-chunk-similarity stratification — and use it to localise where the multi-chunk dependence holds. On a stratified sample (n = 250), 69.8% of items not answerable closed-book cannot be answered from any strict subset of their supporting chunks (strict-necessity violation rate 15.2%). A human annotation study finds the verifier reliable (Cohen’s ąppa = 0.903) and VERGE items higher in QA integrity and distractor quality than the closest prior method (p < 0.001).</abstract>
<identifier type="citekey">morita-2026-verge</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.139/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>3036</start>
<end>3062</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T VERGE: Verification-Enhanced Generation of Multi-Hop Evaluation Datasets for Task-Specific RAG
%A Morita, Kyosuke
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F morita-2026-verge
%X Evaluating retrieval-augmented generation (RAG) on custom knowledge bases requires task-specific benchmarks of multi-chunk evidence synthesis, yet most existing generation pipelines apply quality control only after the full question set has been solved. We present VERGE, which moves quality control inside the generation loop: a verification agent targets, per item and without an external solver pass, an operational multi-chunk criterion and a QA-integrity criterion, refining items that fail and discarding those it cannot repair. From 11,052 candidates across six domains, VERGE yields 2,854 verified multiple-choice items targeting aggregation- and comparison-style synthesis over topically coherent evidence. We contribute a method-agnostic validation protocol — exhaustive strict-subset ablation, chunk-removal ablation, format-unconfounded open-ended re-evaluation, and inter-chunk-similarity stratification — and use it to localise where the multi-chunk dependence holds. On a stratified sample (n = 250), 69.8% of items not answerable closed-book cannot be answered from any strict subset of their supporting chunks (strict-necessity violation rate 15.2%). A human annotation study finds the verifier reliable (Cohen’s ąppa = 0.903) and VERGE items higher in QA integrity and distractor quality than the closest prior method (p < 0.001).
%U https://aclanthology.org/2026.aacl-long.139/
%P 3036-3062
Markdown (Informal)
[VERGE: Verification-Enhanced Generation of Multi-Hop Evaluation Datasets for Task-Specific RAG](https://aclanthology.org/2026.aacl-long.139/) (Morita, AACL-IJCNLP 2026)
ACL