@inproceedings{rohlfs-etal-2026-evaluators,
title = "When Evaluators Cry Wolf: Lessons from Production {LLM}-as-Judge Evaluation in Educational {AI}",
author = "Rohlfs, Chris and
Bosse, Rodrigo Vergara and
Hopper, Priscilla and
O{'}Keefe, Keanon and
Russell, Patrick J",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.193/",
pages = "3240--3255",
ISBN = "979-8-89176-460-6",
abstract = "MagicSchool{'}s K{--}12 AI product suite is used by millions of teachers and serves millions of teacher and student messages each month. Our team monitors output along four priority dimensions{---}student safety, tone and instructional role, pedagogical value, and structural output quality{---}and tracks how often it fails on each. As our program matured, false positives came to dominate the evaluators' flags, misdirecting scarce analyst attention away from failures that warrant product change. To address this, we deployed three enhancements{---}unanimous-fail panels of repeated judge runs, per-evaluator judge-model choices, and softened rubrics{---}backed by a pair of synthetic datasets: a benchmark that measures how often cases are flagged, and an egregious-failure set as a check that severe cases still fail. That set is human-reviewed, with 12 to 79 cases per evaluator, each designed to exhibit an unambiguous violation of the failure mode it targets. Across 21 deployed evaluators, final configurations reached a median benchmark activation rate of 0.04{\%} (16 of 21 at or below 0.2{\%}; 8 at 0.00{\%}). The strategy cuts confirmed false-positive flags by 99{\%} and raises per-flag precision from 0.6{\%} to 49{\%}, while holding egregious-failure capture at 100{\%} and improving it on 8 of 21 evaluators."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="rohlfs-etal-2026-evaluators">
<titleInfo>
<title>When Evaluators Cry Wolf: Lessons from Production LLM-as-Judge Evaluation in Educational AI</title>
</titleInfo>
<name type="personal">
<namePart type="given">Chris</namePart>
<namePart type="family">Rohlfs</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rodrigo</namePart>
<namePart type="given">Vergara</namePart>
<namePart type="family">Bosse</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Priscilla</namePart>
<namePart type="family">Hopper</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keanon</namePart>
<namePart type="family">O’Keefe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Patrick</namePart>
<namePart type="given">J</namePart>
<namePart type="family">Russell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>MagicSchool’s K–12 AI product suite is used by millions of teachers and serves millions of teacher and student messages each month. Our team monitors output along four priority dimensions—student safety, tone and instructional role, pedagogical value, and structural output quality—and tracks how often it fails on each. As our program matured, false positives came to dominate the evaluators’ flags, misdirecting scarce analyst attention away from failures that warrant product change. To address this, we deployed three enhancements—unanimous-fail panels of repeated judge runs, per-evaluator judge-model choices, and softened rubrics—backed by a pair of synthetic datasets: a benchmark that measures how often cases are flagged, and an egregious-failure set as a check that severe cases still fail. That set is human-reviewed, with 12 to 79 cases per evaluator, each designed to exhibit an unambiguous violation of the failure mode it targets. Across 21 deployed evaluators, final configurations reached a median benchmark activation rate of 0.04% (16 of 21 at or below 0.2%; 8 at 0.00%). The strategy cuts confirmed false-positive flags by 99% and raises per-flag precision from 0.6% to 49%, while holding egregious-failure capture at 100% and improving it on 8 of 21 evaluators.</abstract>
<identifier type="citekey">rohlfs-etal-2026-evaluators</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.193/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>3240</start>
<end>3255</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Evaluators Cry Wolf: Lessons from Production LLM-as-Judge Evaluation in Educational AI
%A Rohlfs, Chris
%A Bosse, Rodrigo Vergara
%A Hopper, Priscilla
%A O’Keefe, Keanon
%A Russell, Patrick J.
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F rohlfs-etal-2026-evaluators
%X MagicSchool’s K–12 AI product suite is used by millions of teachers and serves millions of teacher and student messages each month. Our team monitors output along four priority dimensions—student safety, tone and instructional role, pedagogical value, and structural output quality—and tracks how often it fails on each. As our program matured, false positives came to dominate the evaluators’ flags, misdirecting scarce analyst attention away from failures that warrant product change. To address this, we deployed three enhancements—unanimous-fail panels of repeated judge runs, per-evaluator judge-model choices, and softened rubrics—backed by a pair of synthetic datasets: a benchmark that measures how often cases are flagged, and an egregious-failure set as a check that severe cases still fail. That set is human-reviewed, with 12 to 79 cases per evaluator, each designed to exhibit an unambiguous violation of the failure mode it targets. Across 21 deployed evaluators, final configurations reached a median benchmark activation rate of 0.04% (16 of 21 at or below 0.2%; 8 at 0.00%). The strategy cuts confirmed false-positive flags by 99% and raises per-flag precision from 0.6% to 49%, while holding egregious-failure capture at 100% and improving it on 8 of 21 evaluators.
%U https://aclanthology.org/2026.emnlp-industry.193/
%P 3240-3255
Markdown (Informal)
[When Evaluators Cry Wolf: Lessons from Production LLM-as-Judge Evaluation in Educational AI](https://aclanthology.org/2026.emnlp-industry.193/) (Rohlfs et al., EMNLP 2026)
ACL