@inproceedings{he-etal-2026-agreement,
title = "When Agreement Is Not Enough: Evidence-Risk Arbitration for Multi-Strategy Agentic {QA}",
author = "He, Yang and
Wang, Jinlin and
Ji, Yulong and
Zhang, Jianwei",
editor = "Kamalloo, Ehsan and
Gontier, Nicolas and
Rozanov, Nikolai and
Hashemzadeh, Maryam and
Arora, Akhil",
booktitle = "Proceedings of the Second Workshop for {RE}search on Agent Language Models ({REALM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.realm-1.86/",
pages = "1236--1242",
ISBN = "979-8-89176-485-9",
abstract = "Multi-strategy question-answering agents often produce several plausible responses to one query, but must ultimately show the user only one. Voting treats answer agreement as trust, yet strategies can disagree for different reasons, and identical answers can rest on evidence of unequal quality. We cast this final choice as diagnosis-first evidence-risk arbitration and introduce the Evidence-Risk-Aware Strategy Reranker (ERSR), which canonicalizes a pool of six responses, classifies their agreement type, and applies a type-specific evidence rule while returning one unchanged response. Calibration uses only development data; for each evaluation item, trusted reference answers, trusted reference evidence, and utility stay hidden, and the selector never abstains. On a 900-item development split drawn from FEVER, HotpotQA, and SciFact, ERSR improves evidence-aware utility over a development-selected fixed strategy by +0.052, with the largest gains on label conflicts and unanimous answers with uneven evidence. These results suggest that response-set structure, not agreement alone, is a useful signal for choosing trustworthy answers."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="he-etal-2026-agreement">
<titleInfo>
<title>When Agreement Is Not Enough: Evidence-Risk Arbitration for Multi-Strategy Agentic QA</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yang</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jinlin</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yulong</namePart>
<namePart type="family">Ji</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jianwei</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ehsan</namePart>
<namePart type="family">Kamalloo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicolas</namePart>
<namePart type="family">Gontier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikolai</namePart>
<namePart type="family">Rozanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maryam</namePart>
<namePart type="family">Hashemzadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akhil</namePart>
<namePart type="family">Arora</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-485-9</identifier>
</relatedItem>
<abstract>Multi-strategy question-answering agents often produce several plausible responses to one query, but must ultimately show the user only one. Voting treats answer agreement as trust, yet strategies can disagree for different reasons, and identical answers can rest on evidence of unequal quality. We cast this final choice as diagnosis-first evidence-risk arbitration and introduce the Evidence-Risk-Aware Strategy Reranker (ERSR), which canonicalizes a pool of six responses, classifies their agreement type, and applies a type-specific evidence rule while returning one unchanged response. Calibration uses only development data; for each evaluation item, trusted reference answers, trusted reference evidence, and utility stay hidden, and the selector never abstains. On a 900-item development split drawn from FEVER, HotpotQA, and SciFact, ERSR improves evidence-aware utility over a development-selected fixed strategy by +0.052, with the largest gains on label conflicts and unanimous answers with uneven evidence. These results suggest that response-set structure, not agreement alone, is a useful signal for choosing trustworthy answers.</abstract>
<identifier type="citekey">he-etal-2026-agreement</identifier>
<location>
<url>https://aclanthology.org/2026.realm-1.86/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1236</start>
<end>1242</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Agreement Is Not Enough: Evidence-Risk Arbitration for Multi-Strategy Agentic QA
%A He, Yang
%A Wang, Jinlin
%A Ji, Yulong
%A Zhang, Jianwei
%Y Kamalloo, Ehsan
%Y Gontier, Nicolas
%Y Rozanov, Nikolai
%Y Hashemzadeh, Maryam
%Y Arora, Akhil
%S Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-485-9
%F he-etal-2026-agreement
%X Multi-strategy question-answering agents often produce several plausible responses to one query, but must ultimately show the user only one. Voting treats answer agreement as trust, yet strategies can disagree for different reasons, and identical answers can rest on evidence of unequal quality. We cast this final choice as diagnosis-first evidence-risk arbitration and introduce the Evidence-Risk-Aware Strategy Reranker (ERSR), which canonicalizes a pool of six responses, classifies their agreement type, and applies a type-specific evidence rule while returning one unchanged response. Calibration uses only development data; for each evaluation item, trusted reference answers, trusted reference evidence, and utility stay hidden, and the selector never abstains. On a 900-item development split drawn from FEVER, HotpotQA, and SciFact, ERSR improves evidence-aware utility over a development-selected fixed strategy by +0.052, with the largest gains on label conflicts and unanimous answers with uneven evidence. These results suggest that response-set structure, not agreement alone, is a useful signal for choosing trustworthy answers.
%U https://aclanthology.org/2026.realm-1.86/
%P 1236-1242
Markdown (Informal)
[When Agreement Is Not Enough: Evidence-Risk Arbitration for Multi-Strategy Agentic QA](https://aclanthology.org/2026.realm-1.86/) (He et al., REALM 2026)
ACL