@inproceedings{doan-etal-2026-benchmark,
title = "A Benchmark Construction and Evaluation Framework for Specialist Domains: Case Study on Defense-Related Documents",
author = "Doan, Bao Gia and
Joshi, Aditya and
Elinas, Pantelis and
Bodhankar, Aarya and
Leslie, Oscar and
Marchant, Tom and
Salim, Flora D.",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.944/",
pages = "21747--21775",
ISBN = "979-8-89176-459-0",
abstract = "RAG-based question-answering (QA) in specialist domains faces a cold-start problem: no evaluative benchmark and no labeled data for post-training. We present DoRA (Domain-oriented RAG Assessment), a benchmark construction and evaluation framework that needs only a small set of specialist-domain documents, integrating and expert-validating otherwise established components for this cold-start setting. DoRA generates synthetic QA training and evaluation sets with auditable evidence across five domain-specific intents. To mitigate same-pipeline circularity, its training and test splits use different LLM families (Claude Sonnet for training; GPT-4o for test) drawn from disjoint seed-document corpora. Instantiated on 40 English-language defense documents, DoRA yields approximately 6.6K curated instances. Against 7 LLM baselines over a 1,259-sample benchmark, a LoRA-adapted Llama-3.1-8B trained on the synthetic split improves all 6 coverage and faithfulness metrics{---}each gain significant under a paired bootstrap test{---}and reduces hallucination by more than half under the default GTE retrieval setting, with gains persisting across alternative retrievers, prompting-based baselines, and matched answer-length budgets. Defense experts validate the synthetic QA, calibrate the LLM judge, and supply a fully human-written test set. We position DoRA as a practical framework for specialist-domain RAG under domain shift, with defense as a high-stakes case study."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="doan-etal-2026-benchmark">
<titleInfo>
<title>A Benchmark Construction and Evaluation Framework for Specialist Domains: Case Study on Defense-Related Documents</title>
</titleInfo>
<name type="personal">
<namePart type="given">Bao</namePart>
<namePart type="given">Gia</namePart>
<namePart type="family">Doan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aditya</namePart>
<namePart type="family">Joshi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pantelis</namePart>
<namePart type="family">Elinas</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aarya</namePart>
<namePart type="family">Bodhankar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Oscar</namePart>
<namePart type="family">Leslie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tom</namePart>
<namePart type="family">Marchant</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Flora</namePart>
<namePart type="given">D</namePart>
<namePart type="family">Salim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>RAG-based question-answering (QA) in specialist domains faces a cold-start problem: no evaluative benchmark and no labeled data for post-training. We present DoRA (Domain-oriented RAG Assessment), a benchmark construction and evaluation framework that needs only a small set of specialist-domain documents, integrating and expert-validating otherwise established components for this cold-start setting. DoRA generates synthetic QA training and evaluation sets with auditable evidence across five domain-specific intents. To mitigate same-pipeline circularity, its training and test splits use different LLM families (Claude Sonnet for training; GPT-4o for test) drawn from disjoint seed-document corpora. Instantiated on 40 English-language defense documents, DoRA yields approximately 6.6K curated instances. Against 7 LLM baselines over a 1,259-sample benchmark, a LoRA-adapted Llama-3.1-8B trained on the synthetic split improves all 6 coverage and faithfulness metrics—each gain significant under a paired bootstrap test—and reduces hallucination by more than half under the default GTE retrieval setting, with gains persisting across alternative retrievers, prompting-based baselines, and matched answer-length budgets. Defense experts validate the synthetic QA, calibrate the LLM judge, and supply a fully human-written test set. We position DoRA as a practical framework for specialist-domain RAG under domain shift, with defense as a high-stakes case study.</abstract>
<identifier type="citekey">doan-etal-2026-benchmark</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.944/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>21747</start>
<end>21775</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T A Benchmark Construction and Evaluation Framework for Specialist Domains: Case Study on Defense-Related Documents
%A Doan, Bao Gia
%A Joshi, Aditya
%A Elinas, Pantelis
%A Bodhankar, Aarya
%A Leslie, Oscar
%A Marchant, Tom
%A Salim, Flora D.
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F doan-etal-2026-benchmark
%X RAG-based question-answering (QA) in specialist domains faces a cold-start problem: no evaluative benchmark and no labeled data for post-training. We present DoRA (Domain-oriented RAG Assessment), a benchmark construction and evaluation framework that needs only a small set of specialist-domain documents, integrating and expert-validating otherwise established components for this cold-start setting. DoRA generates synthetic QA training and evaluation sets with auditable evidence across five domain-specific intents. To mitigate same-pipeline circularity, its training and test splits use different LLM families (Claude Sonnet for training; GPT-4o for test) drawn from disjoint seed-document corpora. Instantiated on 40 English-language defense documents, DoRA yields approximately 6.6K curated instances. Against 7 LLM baselines over a 1,259-sample benchmark, a LoRA-adapted Llama-3.1-8B trained on the synthetic split improves all 6 coverage and faithfulness metrics—each gain significant under a paired bootstrap test—and reduces hallucination by more than half under the default GTE retrieval setting, with gains persisting across alternative retrievers, prompting-based baselines, and matched answer-length budgets. Defense experts validate the synthetic QA, calibrate the LLM judge, and supply a fully human-written test set. We position DoRA as a practical framework for specialist-domain RAG under domain shift, with defense as a high-stakes case study.
%U https://aclanthology.org/2026.emnlp-main.944/
%P 21747-21775
Markdown (Informal)
[A Benchmark Construction and Evaluation Framework for Specialist Domains: Case Study on Defense-Related Documents](https://aclanthology.org/2026.emnlp-main.944/) (Doan et al., EMNLP 2026)
ACL