@inproceedings{sayana-etal-2026-compass,
title = "{COMPASS}: Critique-guided Optimization of {M}eta{P}olicies for Agentic Systems with Reinforcement Learning",
author = "Sayana, Krishna and
Todi, Ketan Kumar and
Jash, Ambarish and
Sodhi, Sukhdeep",
editor = "Kamalloo, Ehsan and
Gontier, Nicolas and
Rozanov, Nikolai and
Hashemzadeh, Maryam and
Arora, Akhil",
booktitle = "Proceedings of the Second Workshop for {RE}search on Agent Language Models ({REALM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.realm-1.77/",
pages = "1118--1143",
ISBN = "979-8-89176-485-9",
abstract = "As large language models are increasingly deployed as autonomous agents, optimizing their behavior in agentic reasoning and tool-use environments remains challenging due to the stochasticity of environment trajectories and associated high reward latencies. Existing automated prompt optimization (APO) paradigms struggle in these regimes: discrete evolutionary methods (e.g., GEPA, ACE) lack parametric memory, while instance-level reinforcement learning (RL) approaches suffer from severe gradient variance and reward collapse on small, human-curated datasets. To address these limitations, we introduce COMPASS (Critique-guided Optimization of MetaPolicies for Agentic SystemS), a sample-efficient dual-agent framework that shifts optimization toward task-level metapolicies and demonstrates that an RL-based prompter policy can be effectively trained on such tasks. To overcome sparse feedback, COMPASS introduces a Critique Experience Buffer that couples scalar rewards with dense textual critiques. This mechanism allows an active ``Prompter'' policy to internalize corrective linguistic patterns and amortize experiential self-reflection directly into its weights. Empirical evaluations demonstrate that COMPASS stabilizes training and yields up to $2.4\times$ faster convergence. On complex reasoning (BBEH) and tool-use ($\tau$-bench) benchmarks, our framework improves success rates from 55{\%} to 90{\%} and 74{\%} to 91{\%}, respectively, while naturally discovering specialized algorithmic strategies."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sayana-etal-2026-compass">
<titleInfo>
<title>COMPASS: Critique-guided Optimization of MetaPolicies for Agentic Systems with Reinforcement Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Krishna</namePart>
<namePart type="family">Sayana</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ketan</namePart>
<namePart type="given">Kumar</namePart>
<namePart type="family">Todi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ambarish</namePart>
<namePart type="family">Jash</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sukhdeep</namePart>
<namePart type="family">Sodhi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ehsan</namePart>
<namePart type="family">Kamalloo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicolas</namePart>
<namePart type="family">Gontier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikolai</namePart>
<namePart type="family">Rozanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maryam</namePart>
<namePart type="family">Hashemzadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akhil</namePart>
<namePart type="family">Arora</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-485-9</identifier>
</relatedItem>
<abstract>As large language models are increasingly deployed as autonomous agents, optimizing their behavior in agentic reasoning and tool-use environments remains challenging due to the stochasticity of environment trajectories and associated high reward latencies. Existing automated prompt optimization (APO) paradigms struggle in these regimes: discrete evolutionary methods (e.g., GEPA, ACE) lack parametric memory, while instance-level reinforcement learning (RL) approaches suffer from severe gradient variance and reward collapse on small, human-curated datasets. To address these limitations, we introduce COMPASS (Critique-guided Optimization of MetaPolicies for Agentic SystemS), a sample-efficient dual-agent framework that shifts optimization toward task-level metapolicies and demonstrates that an RL-based prompter policy can be effectively trained on such tasks. To overcome sparse feedback, COMPASS introduces a Critique Experience Buffer that couples scalar rewards with dense textual critiques. This mechanism allows an active “Prompter” policy to internalize corrective linguistic patterns and amortize experiential self-reflection directly into its weights. Empirical evaluations demonstrate that COMPASS stabilizes training and yields up to 2.4\times faster convergence. On complex reasoning (BBEH) and tool-use (τ-bench) benchmarks, our framework improves success rates from 55% to 90% and 74% to 91%, respectively, while naturally discovering specialized algorithmic strategies.</abstract>
<identifier type="citekey">sayana-etal-2026-compass</identifier>
<location>
<url>https://aclanthology.org/2026.realm-1.77/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1118</start>
<end>1143</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T COMPASS: Critique-guided Optimization of MetaPolicies for Agentic Systems with Reinforcement Learning
%A Sayana, Krishna
%A Todi, Ketan Kumar
%A Jash, Ambarish
%A Sodhi, Sukhdeep
%Y Kamalloo, Ehsan
%Y Gontier, Nicolas
%Y Rozanov, Nikolai
%Y Hashemzadeh, Maryam
%Y Arora, Akhil
%S Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-485-9
%F sayana-etal-2026-compass
%X As large language models are increasingly deployed as autonomous agents, optimizing their behavior in agentic reasoning and tool-use environments remains challenging due to the stochasticity of environment trajectories and associated high reward latencies. Existing automated prompt optimization (APO) paradigms struggle in these regimes: discrete evolutionary methods (e.g., GEPA, ACE) lack parametric memory, while instance-level reinforcement learning (RL) approaches suffer from severe gradient variance and reward collapse on small, human-curated datasets. To address these limitations, we introduce COMPASS (Critique-guided Optimization of MetaPolicies for Agentic SystemS), a sample-efficient dual-agent framework that shifts optimization toward task-level metapolicies and demonstrates that an RL-based prompter policy can be effectively trained on such tasks. To overcome sparse feedback, COMPASS introduces a Critique Experience Buffer that couples scalar rewards with dense textual critiques. This mechanism allows an active “Prompter” policy to internalize corrective linguistic patterns and amortize experiential self-reflection directly into its weights. Empirical evaluations demonstrate that COMPASS stabilizes training and yields up to 2.4\times faster convergence. On complex reasoning (BBEH) and tool-use (τ-bench) benchmarks, our framework improves success rates from 55% to 90% and 74% to 91%, respectively, while naturally discovering specialized algorithmic strategies.
%U https://aclanthology.org/2026.realm-1.77/
%P 1118-1143
Markdown (Informal)
[COMPASS: Critique-guided Optimization of MetaPolicies for Agentic Systems with Reinforcement Learning](https://aclanthology.org/2026.realm-1.77/) (Sayana et al., REALM 2026)
ACL