@inproceedings{bajpai-etal-2026-exploring,
title = "Exploring Capability Thresholds in Ultra-Lightweight {LLM} Judges for Nugget-Based Report Evaluation",
author = "Bajpai, Mann and
Chatwal, Pulkit and
Deswal, Priyanshu and
Singh, Harish Pratap and
Mishra, Santosh Kumar",
editor = "Yang, Eugene and
Lawrie, Dawn and
MacAvaney, Sean and
Mayfield, James and
Soldaini, Luca and
Yates, Andrew",
booktitle = "Proceedings of the 1st Workshop on Multilingual Report Generation via Retrieval Augmented Generation ({RAG}4{R}eports 2026)",
month = jul,
year = "2026",
address = "San Diego, CA, USA",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.rag4reports-1.13/",
pages = "94--98",
ISBN = "979-8-89176-417-0",
abstract = "Reliable automatic evaluation of retrieval-grounded long-form reports typically requires human annotation or frontier-scale proprietary LLMs, both of which are expensive in constrained settings. Team rgipt participated in RAG4Reports@ACL 2026 Task 1 with a zero-shot nugget-verification system that runs entirely on a single NVIDIA T4 GPU. We compare three ultra-lightweight decoder-only models: Qwen2-0.5B, Qwen2-1.5B, and Qwen2.5-0.5B, under identical inference conditions to examine how small an LLM judge can be while retaining human-aligned ranking signal. Both Qwen2 models produced negative $\tau_{\text{gap}}$, whereas Qwen2.5-0.5B achieved $\tau_{\text{gap}} = 0.0772$ and Pearson $r = 0.2209$, ranking 13th of 21 teams. Within this family and evaluation setting, model generation appears to matter more than parameter count, although this finding is based on three configurations on a single task and warrants further validation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="bajpai-etal-2026-exploring">
<titleInfo>
<title>Exploring Capability Thresholds in Ultra-Lightweight LLM Judges for Nugget-Based Report Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mann</namePart>
<namePart type="family">Bajpai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pulkit</namePart>
<namePart type="family">Chatwal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Priyanshu</namePart>
<namePart type="family">Deswal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Harish</namePart>
<namePart type="given">Pratap</namePart>
<namePart type="family">Singh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Santosh</namePart>
<namePart type="given">Kumar</namePart>
<namePart type="family">Mishra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Multilingual Report Generation via Retrieval Augmented Generation (RAG4Reports 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Eugene</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dawn</namePart>
<namePart type="family">Lawrie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sean</namePart>
<namePart type="family">MacAvaney</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">James</namePart>
<namePart type="family">Mayfield</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Luca</namePart>
<namePart type="family">Soldaini</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andrew</namePart>
<namePart type="family">Yates</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">San Diego, CA, USA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-417-0</identifier>
</relatedItem>
<abstract>Reliable automatic evaluation of retrieval-grounded long-form reports typically requires human annotation or frontier-scale proprietary LLMs, both of which are expensive in constrained settings. Team rgipt participated in RAG4Reports@ACL 2026 Task 1 with a zero-shot nugget-verification system that runs entirely on a single NVIDIA T4 GPU. We compare three ultra-lightweight decoder-only models: Qwen2-0.5B, Qwen2-1.5B, and Qwen2.5-0.5B, under identical inference conditions to examine how small an LLM judge can be while retaining human-aligned ranking signal. Both Qwen2 models produced negative τ_\textgap, whereas Qwen2.5-0.5B achieved τ_\textgap = 0.0772 and Pearson r = 0.2209, ranking 13th of 21 teams. Within this family and evaluation setting, model generation appears to matter more than parameter count, although this finding is based on three configurations on a single task and warrants further validation.</abstract>
<identifier type="citekey">bajpai-etal-2026-exploring</identifier>
<location>
<url>https://aclanthology.org/2026.rag4reports-1.13/</url>
</location>
<part>
<date>2026-07</date>
<extent unit="page">
<start>94</start>
<end>98</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Exploring Capability Thresholds in Ultra-Lightweight LLM Judges for Nugget-Based Report Evaluation
%A Bajpai, Mann
%A Chatwal, Pulkit
%A Deswal, Priyanshu
%A Singh, Harish Pratap
%A Mishra, Santosh Kumar
%Y Yang, Eugene
%Y Lawrie, Dawn
%Y MacAvaney, Sean
%Y Mayfield, James
%Y Soldaini, Luca
%Y Yates, Andrew
%S Proceedings of the 1st Workshop on Multilingual Report Generation via Retrieval Augmented Generation (RAG4Reports 2026)
%D 2026
%8 July
%I Association for Computational Linguistics
%C San Diego, CA, USA
%@ 979-8-89176-417-0
%F bajpai-etal-2026-exploring
%X Reliable automatic evaluation of retrieval-grounded long-form reports typically requires human annotation or frontier-scale proprietary LLMs, both of which are expensive in constrained settings. Team rgipt participated in RAG4Reports@ACL 2026 Task 1 with a zero-shot nugget-verification system that runs entirely on a single NVIDIA T4 GPU. We compare three ultra-lightweight decoder-only models: Qwen2-0.5B, Qwen2-1.5B, and Qwen2.5-0.5B, under identical inference conditions to examine how small an LLM judge can be while retaining human-aligned ranking signal. Both Qwen2 models produced negative τ_\textgap, whereas Qwen2.5-0.5B achieved τ_\textgap = 0.0772 and Pearson r = 0.2209, ranking 13th of 21 teams. Within this family and evaluation setting, model generation appears to matter more than parameter count, although this finding is based on three configurations on a single task and warrants further validation.
%U https://aclanthology.org/2026.rag4reports-1.13/
%P 94-98
Markdown (Informal)
[Exploring Capability Thresholds in Ultra-Lightweight LLM Judges for Nugget-Based Report Evaluation](https://aclanthology.org/2026.rag4reports-1.13/) (Bajpai et al., RAG4Reports 2026)
ACL