@inproceedings{harish-2026-two,
title = "Two Thirds of the Best-of-K Oracle Band in Tool Agents Is Unreachable by Reward-Free Selection",
author = "Harish, Srinivas",
editor = "Kamalloo, Ehsan and
Gontier, Nicolas and
Rozanov, Nikolai and
Hashemzadeh, Maryam and
Arora, Akhil",
booktitle = "Proceedings of the Second Workshop for {RE}search on Agent Language Models ({REALM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.realm-1.39/",
pages = "571--578",
ISBN = "979-8-89176-485-9",
abstract = "Running a tool agent n times and keeping the best run is worth 2.19{--}22.37 points on {\ensuremath{\tau}}{\texttwosuperior}-bench, a floor any scaffold spending n times the calls has to clear. Choosing among those runs, for no additional model calls, is worth anywhere from {\ensuremath{-}}23.68 to +12.28 depending on which rule does the choosing. We evaluate 9 selectors that read only the agent{'}s own trajectory across 28 model {\texttimes} domain cells and 10 403 released trials, and add an LLM judge that reads the trajectories but never a reward and never the evaluation criteria. Every arm is therefore reward-free: the nine rules are verifier-free as well, the judge is not, and neither sees a label. Against a mean best-of-K oracle band of 12.29 points, which is what a selector holding the reward would collect, the judge recovers 32.7{\%} and the best free rule 21.0{\%}, leaving 67.3{\%} of that band unreachable without the label. One measurable property of the rollouts orders the task distributions, the no-mode rate, the share of tasks on which no two of the K rollouts emit the same write-action sequence, and what it predicts is when agreement is exploitable rather than when selection is possible. Where agreement-based rules are nearly inert the judge is worth +5.95 against +0.53; where the rollouts are undifferentiated it returns 1.19 against a band of 14.4 points; on a third domain it returns {\ensuremath{-}}0.64 and loses to a free heuristic costing one fewer call."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="harish-2026-two">
<titleInfo>
<title>Two Thirds of the Best-of-K Oracle Band in Tool Agents Is Unreachable by Reward-Free Selection</title>
</titleInfo>
<name type="personal">
<namePart type="given">Srinivas</namePart>
<namePart type="family">Harish</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ehsan</namePart>
<namePart type="family">Kamalloo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicolas</namePart>
<namePart type="family">Gontier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikolai</namePart>
<namePart type="family">Rozanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maryam</namePart>
<namePart type="family">Hashemzadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akhil</namePart>
<namePart type="family">Arora</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-485-9</identifier>
</relatedItem>
<abstract>Running a tool agent n times and keeping the best run is worth 2.19–22.37 points on \ensuremathτ²-bench, a floor any scaffold spending n times the calls has to clear. Choosing among those runs, for no additional model calls, is worth anywhere from \ensuremath-23.68 to +12.28 depending on which rule does the choosing. We evaluate 9 selectors that read only the agent’s own trajectory across 28 model × domain cells and 10 403 released trials, and add an LLM judge that reads the trajectories but never a reward and never the evaluation criteria. Every arm is therefore reward-free: the nine rules are verifier-free as well, the judge is not, and neither sees a label. Against a mean best-of-K oracle band of 12.29 points, which is what a selector holding the reward would collect, the judge recovers 32.7% and the best free rule 21.0%, leaving 67.3% of that band unreachable without the label. One measurable property of the rollouts orders the task distributions, the no-mode rate, the share of tasks on which no two of the K rollouts emit the same write-action sequence, and what it predicts is when agreement is exploitable rather than when selection is possible. Where agreement-based rules are nearly inert the judge is worth +5.95 against +0.53; where the rollouts are undifferentiated it returns 1.19 against a band of 14.4 points; on a third domain it returns \ensuremath-0.64 and loses to a free heuristic costing one fewer call.</abstract>
<identifier type="citekey">harish-2026-two</identifier>
<location>
<url>https://aclanthology.org/2026.realm-1.39/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>571</start>
<end>578</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Two Thirds of the Best-of-K Oracle Band in Tool Agents Is Unreachable by Reward-Free Selection
%A Harish, Srinivas
%Y Kamalloo, Ehsan
%Y Gontier, Nicolas
%Y Rozanov, Nikolai
%Y Hashemzadeh, Maryam
%Y Arora, Akhil
%S Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-485-9
%F harish-2026-two
%X Running a tool agent n times and keeping the best run is worth 2.19–22.37 points on \ensuremathτ²-bench, a floor any scaffold spending n times the calls has to clear. Choosing among those runs, for no additional model calls, is worth anywhere from \ensuremath-23.68 to +12.28 depending on which rule does the choosing. We evaluate 9 selectors that read only the agent’s own trajectory across 28 model × domain cells and 10 403 released trials, and add an LLM judge that reads the trajectories but never a reward and never the evaluation criteria. Every arm is therefore reward-free: the nine rules are verifier-free as well, the judge is not, and neither sees a label. Against a mean best-of-K oracle band of 12.29 points, which is what a selector holding the reward would collect, the judge recovers 32.7% and the best free rule 21.0%, leaving 67.3% of that band unreachable without the label. One measurable property of the rollouts orders the task distributions, the no-mode rate, the share of tasks on which no two of the K rollouts emit the same write-action sequence, and what it predicts is when agreement is exploitable rather than when selection is possible. Where agreement-based rules are nearly inert the judge is worth +5.95 against +0.53; where the rollouts are undifferentiated it returns 1.19 against a band of 14.4 points; on a third domain it returns \ensuremath-0.64 and loses to a free heuristic costing one fewer call.
%U https://aclanthology.org/2026.realm-1.39/
%P 571-578
Markdown (Informal)
[Two Thirds of the Best-of-K Oracle Band in Tool Agents Is Unreachable by Reward-Free Selection](https://aclanthology.org/2026.realm-1.39/) (Harish, REALM 2026)
ACL