@inproceedings{dezaki-etal-2026-critics,
title = "When Critics Go Stale: Benchmarking Safety Guardrail Degradation Under Environment Drift",
author = "Dezaki, Fatemeh Taheri and
Kolasani, Lakshman and
Bhaumik, Ankita and
Rao, Siddartha Sathyamoorthy and
Jain, Ayush",
editor = "Kamalloo, Ehsan and
Gontier, Nicolas and
Rozanov, Nikolai and
Hashemzadeh, Maryam and
Arora, Akhil",
booktitle = "Proceedings of the Second Workshop for {RE}search on Agent Language Models ({REALM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.realm-1.63/",
pages = "922--929",
ISBN = "979-8-89176-485-9",
abstract = "LLM-based critics are widely deployed as safety guardrails for agentic systems, yet their reliability under realistic operating conditions remains underexplored. We demonstrate that critics calibrated against a fixed system snapshot degrade as tools are added and application domains expand, with no external signal indicating weakened detection capability. We formalize this problem along controlled axes and introduce DRIFT (Detecting Reliability Issues From Tool-scaling), a benchmark for measuring critic robustness under environment change. Building on SafeToolBench{'}s multi-domain tool ecosystem, we construct verified cross-app composition scenarios, both violating and compliant, where policy violations emerge only from data flow across application boundaries, and evaluate 9 models across 6 provider families on a two-dimensional scaling matrix separating intra-app noise from cross-app noise. We find that (1) all 9 models degrade under cross-app scaling, with UPR increases ranging from +0.083 to +0.296, (2) tool scaling shifts critics toward reduced discriminability, missed violations rise while false blocks simultaneously fall, and (3) cross-app noise consistently dominates intra-app noise, with cross-app deltas positive for all nine models (sign test p = 0.004) while intra-app deltas are directionless (4 of 9 positive, p = 1.0). DRIFT provides a controlled methodology for measuring critic robustness as tool ecosystems evolve, and our results argue for monitoring UPR and FBR jointly rather than aggregate accuracy alone."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="dezaki-etal-2026-critics">
<titleInfo>
<title>When Critics Go Stale: Benchmarking Safety Guardrail Degradation Under Environment Drift</title>
</titleInfo>
<name type="personal">
<namePart type="given">Fatemeh</namePart>
<namePart type="given">Taheri</namePart>
<namePart type="family">Dezaki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lakshman</namePart>
<namePart type="family">Kolasani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ankita</namePart>
<namePart type="family">Bhaumik</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siddartha</namePart>
<namePart type="given">Sathyamoorthy</namePart>
<namePart type="family">Rao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayush</namePart>
<namePart type="family">Jain</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ehsan</namePart>
<namePart type="family">Kamalloo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicolas</namePart>
<namePart type="family">Gontier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikolai</namePart>
<namePart type="family">Rozanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maryam</namePart>
<namePart type="family">Hashemzadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akhil</namePart>
<namePart type="family">Arora</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-485-9</identifier>
</relatedItem>
<abstract>LLM-based critics are widely deployed as safety guardrails for agentic systems, yet their reliability under realistic operating conditions remains underexplored. We demonstrate that critics calibrated against a fixed system snapshot degrade as tools are added and application domains expand, with no external signal indicating weakened detection capability. We formalize this problem along controlled axes and introduce DRIFT (Detecting Reliability Issues From Tool-scaling), a benchmark for measuring critic robustness under environment change. Building on SafeToolBench’s multi-domain tool ecosystem, we construct verified cross-app composition scenarios, both violating and compliant, where policy violations emerge only from data flow across application boundaries, and evaluate 9 models across 6 provider families on a two-dimensional scaling matrix separating intra-app noise from cross-app noise. We find that (1) all 9 models degrade under cross-app scaling, with UPR increases ranging from +0.083 to +0.296, (2) tool scaling shifts critics toward reduced discriminability, missed violations rise while false blocks simultaneously fall, and (3) cross-app noise consistently dominates intra-app noise, with cross-app deltas positive for all nine models (sign test p = 0.004) while intra-app deltas are directionless (4 of 9 positive, p = 1.0). DRIFT provides a controlled methodology for measuring critic robustness as tool ecosystems evolve, and our results argue for monitoring UPR and FBR jointly rather than aggregate accuracy alone.</abstract>
<identifier type="citekey">dezaki-etal-2026-critics</identifier>
<location>
<url>https://aclanthology.org/2026.realm-1.63/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>922</start>
<end>929</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Critics Go Stale: Benchmarking Safety Guardrail Degradation Under Environment Drift
%A Dezaki, Fatemeh Taheri
%A Kolasani, Lakshman
%A Bhaumik, Ankita
%A Rao, Siddartha Sathyamoorthy
%A Jain, Ayush
%Y Kamalloo, Ehsan
%Y Gontier, Nicolas
%Y Rozanov, Nikolai
%Y Hashemzadeh, Maryam
%Y Arora, Akhil
%S Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-485-9
%F dezaki-etal-2026-critics
%X LLM-based critics are widely deployed as safety guardrails for agentic systems, yet their reliability under realistic operating conditions remains underexplored. We demonstrate that critics calibrated against a fixed system snapshot degrade as tools are added and application domains expand, with no external signal indicating weakened detection capability. We formalize this problem along controlled axes and introduce DRIFT (Detecting Reliability Issues From Tool-scaling), a benchmark for measuring critic robustness under environment change. Building on SafeToolBench’s multi-domain tool ecosystem, we construct verified cross-app composition scenarios, both violating and compliant, where policy violations emerge only from data flow across application boundaries, and evaluate 9 models across 6 provider families on a two-dimensional scaling matrix separating intra-app noise from cross-app noise. We find that (1) all 9 models degrade under cross-app scaling, with UPR increases ranging from +0.083 to +0.296, (2) tool scaling shifts critics toward reduced discriminability, missed violations rise while false blocks simultaneously fall, and (3) cross-app noise consistently dominates intra-app noise, with cross-app deltas positive for all nine models (sign test p = 0.004) while intra-app deltas are directionless (4 of 9 positive, p = 1.0). DRIFT provides a controlled methodology for measuring critic robustness as tool ecosystems evolve, and our results argue for monitoring UPR and FBR jointly rather than aggregate accuracy alone.
%U https://aclanthology.org/2026.realm-1.63/
%P 922-929
Markdown (Informal)
[When Critics Go Stale: Benchmarking Safety Guardrail Degradation Under Environment Drift](https://aclanthology.org/2026.realm-1.63/) (Dezaki et al., REALM 2026)
ACL