@inproceedings{selvam-ghosh-2026-eval,
title = "Eval-Pair Matrix: Answer-Paired Meta-Evaluation of {LLM} Judges for Grounded {RAG}",
author = "Selvam, Sriram and
Ghosh, Anneswa",
editor = "Wang, Yimu and
Choi, Yee Man and
Wu, Di and
Ouyang, Siqi and
Azadani, Mozhgan Nasr and
Fung, Yi R. (May)",
booktitle = "Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently ({G}round{LM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.groundlm-1.43/",
pages = "550--565",
ISBN = "979-8-89176-478-1",
abstract = "LLM-as-a-judge evaluation is widely used for retrieval-augmented generation (RAG), but reusing a model as generator and judge complicates identifying self-leniency. We introduce Eval-Pair Matrix, a controlled meta-evaluation protocol for source-grounded RAG. Starting from GaRAGe, we induce one hidden answer-causal contradiction per record, generate answers from perturbed passages with GPT, Grok, and Gemini, and use the same models as blind judges against the original passages. The experiment contains 300 core records, 897 labeled outputs, and 2,683 verdicts in a crossed 3{\texttimes}3 matrix; the primary analysis uses 275 validated records. We estimate same-model effects by pairing judges on the exact same candidate answer. Diagonal and off-diagonal F1 are similar, and aggregate paired recall is near zero for the three tested endpoints ({\ensuremath{-}}0.5 pp; 95{\%} cluster-bootstrap CI [{\ensuremath{-}}2.7,+1.7]). Matching judges flag answers that avoided the induced claim less often ({\ensuremath{-}}4.3 pp). A targeted human evaluation classifies reviewed apparent false positives as alternate source-error detections, adoption-label mistakes, or unclear cases; none were adjudicated as genuine false alarms. We recommend full matrices, answer-paired effects, behavior strata, and label-task alignment."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="selvam-ghosh-2026-eval">
<titleInfo>
<title>Eval-Pair Matrix: Answer-Paired Meta-Evaluation of LLM Judges for Grounded RAG</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sriram</namePart>
<namePart type="family">Selvam</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anneswa</namePart>
<namePart type="family">Ghosh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yee</namePart>
<namePart type="given">Man</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siqi</namePart>
<namePart type="family">Ouyang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mozhgan</namePart>
<namePart type="given">Nasr</namePart>
<namePart type="family">Azadani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="given">R</namePart>
<namePart type="given">(May)</namePart>
<namePart type="family">Fung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-478-1</identifier>
</relatedItem>
<abstract>LLM-as-a-judge evaluation is widely used for retrieval-augmented generation (RAG), but reusing a model as generator and judge complicates identifying self-leniency. We introduce Eval-Pair Matrix, a controlled meta-evaluation protocol for source-grounded RAG. Starting from GaRAGe, we induce one hidden answer-causal contradiction per record, generate answers from perturbed passages with GPT, Grok, and Gemini, and use the same models as blind judges against the original passages. The experiment contains 300 core records, 897 labeled outputs, and 2,683 verdicts in a crossed 3×3 matrix; the primary analysis uses 275 validated records. We estimate same-model effects by pairing judges on the exact same candidate answer. Diagonal and off-diagonal F1 are similar, and aggregate paired recall is near zero for the three tested endpoints (\ensuremath-0.5 pp; 95% cluster-bootstrap CI [\ensuremath-2.7,+1.7]). Matching judges flag answers that avoided the induced claim less often (\ensuremath-4.3 pp). A targeted human evaluation classifies reviewed apparent false positives as alternate source-error detections, adoption-label mistakes, or unclear cases; none were adjudicated as genuine false alarms. We recommend full matrices, answer-paired effects, behavior strata, and label-task alignment.</abstract>
<identifier type="citekey">selvam-ghosh-2026-eval</identifier>
<location>
<url>https://aclanthology.org/2026.groundlm-1.43/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>550</start>
<end>565</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Eval-Pair Matrix: Answer-Paired Meta-Evaluation of LLM Judges for Grounded RAG
%A Selvam, Sriram
%A Ghosh, Anneswa
%Y Wang, Yimu
%Y Choi, Yee Man
%Y Wu, Di
%Y Ouyang, Siqi
%Y Azadani, Mozhgan Nasr
%Y Fung, Yi R. (May)
%S Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-478-1
%F selvam-ghosh-2026-eval
%X LLM-as-a-judge evaluation is widely used for retrieval-augmented generation (RAG), but reusing a model as generator and judge complicates identifying self-leniency. We introduce Eval-Pair Matrix, a controlled meta-evaluation protocol for source-grounded RAG. Starting from GaRAGe, we induce one hidden answer-causal contradiction per record, generate answers from perturbed passages with GPT, Grok, and Gemini, and use the same models as blind judges against the original passages. The experiment contains 300 core records, 897 labeled outputs, and 2,683 verdicts in a crossed 3×3 matrix; the primary analysis uses 275 validated records. We estimate same-model effects by pairing judges on the exact same candidate answer. Diagonal and off-diagonal F1 are similar, and aggregate paired recall is near zero for the three tested endpoints (\ensuremath-0.5 pp; 95% cluster-bootstrap CI [\ensuremath-2.7,+1.7]). Matching judges flag answers that avoided the induced claim less often (\ensuremath-4.3 pp). A targeted human evaluation classifies reviewed apparent false positives as alternate source-error detections, adoption-label mistakes, or unclear cases; none were adjudicated as genuine false alarms. We recommend full matrices, answer-paired effects, behavior strata, and label-task alignment.
%U https://aclanthology.org/2026.groundlm-1.43/
%P 550-565
Markdown (Informal)
[Eval-Pair Matrix: Answer-Paired Meta-Evaluation of LLM Judges for Grounded RAG](https://aclanthology.org/2026.groundlm-1.43/) (Selvam & Ghosh, GroundLM 2026)
ACL