@inproceedings{ouyang-etal-2026-karma,
title = "Karma: A Composable Framework for Evaluating {LLM} Agents on Stateful Kubernetes Operations",
author = "Ouyang, Junhan and
Liu, Shaw and
Wang, Jinlang and
Zhou, Jiawei",
editor = "Agrawal, Sweta and
Abad, Alberto",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: System Demonstrations",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-demos.55/",
pages = "570--581",
ISBN = "979-8-89176-461-3",
abstract = "Large language model (LLM) agents are increasingly used for infrastructure operations on Kubernetes, yet how to test them efficiently and accurately remains unresolved. Realistic long-horizon evaluations are time-consuming and sometimes error-prone to build, since test cases are usually written from scratch even though components such as setup, state management, and verification logic are duplicated across them. We present KARMA, an open-source framework that composes reusable, parameterized operational cases into stateful, multi-stage Kubernetes workflows. Each case textually specifies a task and a required cluster state, and contains a deterministic oracle. The cluster state is preserved across stages, so long-horizon tasks can be constructed with little additional effort. KARMA also provides controls that add further complexity, namely alternating prompt modes and adversary injection. For cross-stage testing, a regression sweep reruns all previous cases and an LLM judge adjudicates the results, detecting execution inconsistencies or conflicts introduced by agents. Alongside the framework, we release a corpus of 89 cases, 38 adversaries, and 327 workflows across eight systems."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ouyang-etal-2026-karma">
<titleInfo>
<title>Karma: A Composable Framework for Evaluating LLM Agents on Stateful Kubernetes Operations</title>
</titleInfo>
<name type="personal">
<namePart type="given">Junhan</namePart>
<namePart type="family">Ouyang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shaw</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jinlang</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiawei</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: System Demonstrations</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sweta</namePart>
<namePart type="family">Agrawal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alberto</namePart>
<namePart type="family">Abad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-461-3</identifier>
</relatedItem>
<abstract>Large language model (LLM) agents are increasingly used for infrastructure operations on Kubernetes, yet how to test them efficiently and accurately remains unresolved. Realistic long-horizon evaluations are time-consuming and sometimes error-prone to build, since test cases are usually written from scratch even though components such as setup, state management, and verification logic are duplicated across them. We present KARMA, an open-source framework that composes reusable, parameterized operational cases into stateful, multi-stage Kubernetes workflows. Each case textually specifies a task and a required cluster state, and contains a deterministic oracle. The cluster state is preserved across stages, so long-horizon tasks can be constructed with little additional effort. KARMA also provides controls that add further complexity, namely alternating prompt modes and adversary injection. For cross-stage testing, a regression sweep reruns all previous cases and an LLM judge adjudicates the results, detecting execution inconsistencies or conflicts introduced by agents. Alongside the framework, we release a corpus of 89 cases, 38 adversaries, and 327 workflows across eight systems.</abstract>
<identifier type="citekey">ouyang-etal-2026-karma</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-demos.55/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>570</start>
<end>581</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Karma: A Composable Framework for Evaluating LLM Agents on Stateful Kubernetes Operations
%A Ouyang, Junhan
%A Liu, Shaw
%A Wang, Jinlang
%A Zhou, Jiawei
%Y Agrawal, Sweta
%Y Abad, Alberto
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: System Demonstrations
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-461-3
%F ouyang-etal-2026-karma
%X Large language model (LLM) agents are increasingly used for infrastructure operations on Kubernetes, yet how to test them efficiently and accurately remains unresolved. Realistic long-horizon evaluations are time-consuming and sometimes error-prone to build, since test cases are usually written from scratch even though components such as setup, state management, and verification logic are duplicated across them. We present KARMA, an open-source framework that composes reusable, parameterized operational cases into stateful, multi-stage Kubernetes workflows. Each case textually specifies a task and a required cluster state, and contains a deterministic oracle. The cluster state is preserved across stages, so long-horizon tasks can be constructed with little additional effort. KARMA also provides controls that add further complexity, namely alternating prompt modes and adversary injection. For cross-stage testing, a regression sweep reruns all previous cases and an LLM judge adjudicates the results, detecting execution inconsistencies or conflicts introduced by agents. Alongside the framework, we release a corpus of 89 cases, 38 adversaries, and 327 workflows across eight systems.
%U https://aclanthology.org/2026.emnlp-demos.55/
%P 570-581
Markdown (Informal)
[Karma: A Composable Framework for Evaluating LLM Agents on Stateful Kubernetes Operations](https://aclanthology.org/2026.emnlp-demos.55/) (Ouyang et al., EMNLP 2026)
ACL