@inproceedings{zhang-etal-2026-tokencliff,
title = "{T}oken{C}liff: Benchmarking Output-Token Budget Sensitivity in Multi-Turn {LLM} Agents",
author = "Zhang, Bairui and
Liu, Weixuan and
Xue, Defan and
Zhang, Yongqi",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.410/",
pages = "9415--9427",
ISBN = "979-8-89176-459-0",
abstract = "Output-token budgets are a first-order deployment constraint for large language model (LLM) agents, yet existing benchmarks typically evaluate them under a single fixed or generous budget. We introduce TokenCliff, a benchmark for measuring multi-turn agent performance across episode-level output-token budgets. It contains 660 deterministic tasks spanning four capability-oriented domains{---}file operations, data transformation, tool use, and planning{---}plus a compositional multi-step split. We propose two budget-sensitivity metrics: Budget Elasticity, which captures robustness across budget levels, and the Cliff Index, which captures the sharpest adjacent-budget performance drop. Evaluating 10 LLMs at five budget levels from 200 to 8,192 tokens yields 33,000 episodes. Results show that (1) accuracy drops by up to 32 percentage points between adjacent budgets; (2) neither peak performance nor larger model size necessarily implies stronger budget elasticity; and (3) budget sensitivity is substantially greater in the evaluated multi-turn setting than in the single-shot diagnostic. Code and data are available in our public repository: https://github.com/BairuiZhang/TokenCliff"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-tokencliff">
<titleInfo>
<title>TokenCliff: Benchmarking Output-Token Budget Sensitivity in Multi-Turn LLM Agents</title>
</titleInfo>
<name type="personal">
<namePart type="given">Bairui</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weixuan</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Defan</namePart>
<namePart type="family">Xue</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yongqi</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Output-token budgets are a first-order deployment constraint for large language model (LLM) agents, yet existing benchmarks typically evaluate them under a single fixed or generous budget. We introduce TokenCliff, a benchmark for measuring multi-turn agent performance across episode-level output-token budgets. It contains 660 deterministic tasks spanning four capability-oriented domains—file operations, data transformation, tool use, and planning—plus a compositional multi-step split. We propose two budget-sensitivity metrics: Budget Elasticity, which captures robustness across budget levels, and the Cliff Index, which captures the sharpest adjacent-budget performance drop. Evaluating 10 LLMs at five budget levels from 200 to 8,192 tokens yields 33,000 episodes. Results show that (1) accuracy drops by up to 32 percentage points between adjacent budgets; (2) neither peak performance nor larger model size necessarily implies stronger budget elasticity; and (3) budget sensitivity is substantially greater in the evaluated multi-turn setting than in the single-shot diagnostic. Code and data are available in our public repository: https://github.com/BairuiZhang/TokenCliff</abstract>
<identifier type="citekey">zhang-etal-2026-tokencliff</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.410/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>9415</start>
<end>9427</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T TokenCliff: Benchmarking Output-Token Budget Sensitivity in Multi-Turn LLM Agents
%A Zhang, Bairui
%A Liu, Weixuan
%A Xue, Defan
%A Zhang, Yongqi
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F zhang-etal-2026-tokencliff
%X Output-token budgets are a first-order deployment constraint for large language model (LLM) agents, yet existing benchmarks typically evaluate them under a single fixed or generous budget. We introduce TokenCliff, a benchmark for measuring multi-turn agent performance across episode-level output-token budgets. It contains 660 deterministic tasks spanning four capability-oriented domains—file operations, data transformation, tool use, and planning—plus a compositional multi-step split. We propose two budget-sensitivity metrics: Budget Elasticity, which captures robustness across budget levels, and the Cliff Index, which captures the sharpest adjacent-budget performance drop. Evaluating 10 LLMs at five budget levels from 200 to 8,192 tokens yields 33,000 episodes. Results show that (1) accuracy drops by up to 32 percentage points between adjacent budgets; (2) neither peak performance nor larger model size necessarily implies stronger budget elasticity; and (3) budget sensitivity is substantially greater in the evaluated multi-turn setting than in the single-shot diagnostic. Code and data are available in our public repository: https://github.com/BairuiZhang/TokenCliff
%U https://aclanthology.org/2026.emnlp-main.410/
%P 9415-9427
Markdown (Informal)
[TokenCliff: Benchmarking Output-Token Budget Sensitivity in Multi-Turn LLM Agents](https://aclanthology.org/2026.emnlp-main.410/) (Zhang et al., EMNLP 2026)
ACL