@inproceedings{cornacchia-etal-2026-calibrated,
title = "Calibrated Deferral Routing for Cost-Efficient Guardrails",
author = "Cornacchia, Giandomenico and
Padhi, Inkit and
Nagireddy, Manish and
Chaudhury, Subhajit and
Dognin, Pierre and
Pedapati, Tejaswini and
Rawat, Ambrish and
Purcell, Mark and
Varshney, Kush R. and
Sattigeri, Prasanna",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.252/",
pages = "4264--4282",
ISBN = "979-8-89176-460-6",
abstract = "Large Language Models (LLMs) have achieved strong performance across diverse NLP tasks, but deploying them in real-world scenarios raises concerns regarding safety, reliability, and computational cost. To mitigate the risk of harmful, unreliable, or unfaithful content, LLM pipelines increasingly rely on guardrails. Yet deploying multiple guardrails across different tasks and risk categories introduces significant computational and orchestration overhead. In this work, we propose a \textit{calibrated deferral-routing} framework for deploying guardrails both accurately and cost-effectively. Our approach arranges guardrails as a cascade of increasing capability and cost: each input is first evaluated by a lightweight guardrail, while only uncertain instances are deferred to a larger, more capable, and more expensive model. We formalize this routing problem and study practical deferral-calibration strategies by applying post-hoc calibration, including isotonic regression, Platt scaling, and temperature scaling. We evaluate 18 guardrail models from four model families (i.e., Granite Guardian, Llama Guard, ShieldGemma, and GuardReasoner) across 10 harm-detection benchmarks. Our experiments show that: (i) all guardrail models are often poorly calibrated out of the box (ECE ranging from 6{\%} to 28{\%}); (ii) post-hoc calibration substantially improves reliability reducing ECE below 2.5{\%} in most cases; (iii) cascade routing yields large cost savings with minimal accuracy loss, while monotone calibration primarily makes routing thresholds deployable rather than improving rank ordering."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="cornacchia-etal-2026-calibrated">
<titleInfo>
<title>Calibrated Deferral Routing for Cost-Efficient Guardrails</title>
</titleInfo>
<name type="personal">
<namePart type="given">Giandomenico</namePart>
<namePart type="family">Cornacchia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Inkit</namePart>
<namePart type="family">Padhi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Manish</namePart>
<namePart type="family">Nagireddy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Subhajit</namePart>
<namePart type="family">Chaudhury</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pierre</namePart>
<namePart type="family">Dognin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tejaswini</namePart>
<namePart type="family">Pedapati</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ambrish</namePart>
<namePart type="family">Rawat</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mark</namePart>
<namePart type="family">Purcell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kush</namePart>
<namePart type="given">R</namePart>
<namePart type="family">Varshney</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Prasanna</namePart>
<namePart type="family">Sattigeri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>Large Language Models (LLMs) have achieved strong performance across diverse NLP tasks, but deploying them in real-world scenarios raises concerns regarding safety, reliability, and computational cost. To mitigate the risk of harmful, unreliable, or unfaithful content, LLM pipelines increasingly rely on guardrails. Yet deploying multiple guardrails across different tasks and risk categories introduces significant computational and orchestration overhead. In this work, we propose a calibrated deferral-routing framework for deploying guardrails both accurately and cost-effectively. Our approach arranges guardrails as a cascade of increasing capability and cost: each input is first evaluated by a lightweight guardrail, while only uncertain instances are deferred to a larger, more capable, and more expensive model. We formalize this routing problem and study practical deferral-calibration strategies by applying post-hoc calibration, including isotonic regression, Platt scaling, and temperature scaling. We evaluate 18 guardrail models from four model families (i.e., Granite Guardian, Llama Guard, ShieldGemma, and GuardReasoner) across 10 harm-detection benchmarks. Our experiments show that: (i) all guardrail models are often poorly calibrated out of the box (ECE ranging from 6% to 28%); (ii) post-hoc calibration substantially improves reliability reducing ECE below 2.5% in most cases; (iii) cascade routing yields large cost savings with minimal accuracy loss, while monotone calibration primarily makes routing thresholds deployable rather than improving rank ordering.</abstract>
<identifier type="citekey">cornacchia-etal-2026-calibrated</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.252/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>4264</start>
<end>4282</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Calibrated Deferral Routing for Cost-Efficient Guardrails
%A Cornacchia, Giandomenico
%A Padhi, Inkit
%A Nagireddy, Manish
%A Chaudhury, Subhajit
%A Dognin, Pierre
%A Pedapati, Tejaswini
%A Rawat, Ambrish
%A Purcell, Mark
%A Varshney, Kush R.
%A Sattigeri, Prasanna
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F cornacchia-etal-2026-calibrated
%X Large Language Models (LLMs) have achieved strong performance across diverse NLP tasks, but deploying them in real-world scenarios raises concerns regarding safety, reliability, and computational cost. To mitigate the risk of harmful, unreliable, or unfaithful content, LLM pipelines increasingly rely on guardrails. Yet deploying multiple guardrails across different tasks and risk categories introduces significant computational and orchestration overhead. In this work, we propose a calibrated deferral-routing framework for deploying guardrails both accurately and cost-effectively. Our approach arranges guardrails as a cascade of increasing capability and cost: each input is first evaluated by a lightweight guardrail, while only uncertain instances are deferred to a larger, more capable, and more expensive model. We formalize this routing problem and study practical deferral-calibration strategies by applying post-hoc calibration, including isotonic regression, Platt scaling, and temperature scaling. We evaluate 18 guardrail models from four model families (i.e., Granite Guardian, Llama Guard, ShieldGemma, and GuardReasoner) across 10 harm-detection benchmarks. Our experiments show that: (i) all guardrail models are often poorly calibrated out of the box (ECE ranging from 6% to 28%); (ii) post-hoc calibration substantially improves reliability reducing ECE below 2.5% in most cases; (iii) cascade routing yields large cost savings with minimal accuracy loss, while monotone calibration primarily makes routing thresholds deployable rather than improving rank ordering.
%U https://aclanthology.org/2026.emnlp-industry.252/
%P 4264-4282
Markdown (Informal)
[Calibrated Deferral Routing for Cost-Efficient Guardrails](https://aclanthology.org/2026.emnlp-industry.252/) (Cornacchia et al., EMNLP 2026)
ACL
- Giandomenico Cornacchia, Inkit Padhi, Manish Nagireddy, Subhajit Chaudhury, Pierre Dognin, Tejaswini Pedapati, Ambrish Rawat, Mark Purcell, Kush R. Varshney, and Prasanna Sattigeri. 2026. Calibrated Deferral Routing for Cost-Efficient Guardrails. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track, pages 4264–4282, Budapest, Hungary. Association for Computational Linguistics.