@inproceedings{sun-etal-2026-coin,
title = "{C}o{I}n: Counting the Invisible Reasoning Tokens in Commercial Opaque {LLM} {API}s",
author = "Sun, Guoheng and
Wang, Ziyao and
Tian, Bowei and
Liu, Meng and
Shen, Zheyu and
He, Shwai and
He, Yexiao and
Ye, Wanghao and
Wang, Yiting and
Li, Ang",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.363/",
pages = "7594--7620",
ISBN = "979-8-89176-462-0",
abstract = "As post-training techniques evolve, large language models (LLMs) are increasingly augmented with structured multi-step reasoning abilities, often optimized through reinforcement learning. These reasoning-enhanced models outperform standard LLMs on complex tasks and now underpin many commercial LLM APIs. However, to protect proprietary behavior and reduce verbosity, providers typically conceal the reasoning traces while returning only the final answer. This opacity introduces a critical \textbf{transparency gap}: users are billed for invisible reasoning tokens, which often account for the majority of the cost, yet have no means to verify their authenticity. This opens the door to \textit{token count manipulation}, where providers may overreport token usage or inject synthetic, low-effort tokens to inflate charges, a threat that can be carried out at \textit{near-zero cost}. To address this issue, we propose \texttt{CoIn}, a verification framework that audits both the \textit{quantity} and \textit{semantic validity} of hidden tokens. \texttt{CoIn} constructs a verifiable hash tree from token embedding fingerprints to check token counts, and uses embedding-based relevance matching to detect fabricated reasoning content. Experiments demonstrate that \texttt{CoIn}, when deployed as a trusted third-party auditor, can effectively detect token count manipulation with a success rate reaching up to 94.7{\%}, demonstrating a strong ability to restore billing transparency in opaque LLM services. The dataset and code are available at \url{https://github.com/CASE-Lab-UMD/LLM-Auditing-CoIn }."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sun-etal-2026-coin">
<titleInfo>
<title>CoIn: Counting the Invisible Reasoning Tokens in Commercial Opaque LLM APIs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Guoheng</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziyao</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bowei</namePart>
<namePart type="family">Tian</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Meng</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zheyu</namePart>
<namePart type="family">Shen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shwai</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yexiao</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wanghao</namePart>
<namePart type="family">Ye</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yiting</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ang</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>As post-training techniques evolve, large language models (LLMs) are increasingly augmented with structured multi-step reasoning abilities, often optimized through reinforcement learning. These reasoning-enhanced models outperform standard LLMs on complex tasks and now underpin many commercial LLM APIs. However, to protect proprietary behavior and reduce verbosity, providers typically conceal the reasoning traces while returning only the final answer. This opacity introduces a critical transparency gap: users are billed for invisible reasoning tokens, which often account for the majority of the cost, yet have no means to verify their authenticity. This opens the door to token count manipulation, where providers may overreport token usage or inject synthetic, low-effort tokens to inflate charges, a threat that can be carried out at near-zero cost. To address this issue, we propose CoIn, a verification framework that audits both the quantity and semantic validity of hidden tokens. CoIn constructs a verifiable hash tree from token embedding fingerprints to check token counts, and uses embedding-based relevance matching to detect fabricated reasoning content. Experiments demonstrate that CoIn, when deployed as a trusted third-party auditor, can effectively detect token count manipulation with a success rate reaching up to 94.7%, demonstrating a strong ability to restore billing transparency in opaque LLM services. The dataset and code are available at https://github.com/CASE-Lab-UMD/LLM-Auditing-CoIn .</abstract>
<identifier type="citekey">sun-etal-2026-coin</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.363/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>7594</start>
<end>7620</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T CoIn: Counting the Invisible Reasoning Tokens in Commercial Opaque LLM APIs
%A Sun, Guoheng
%A Wang, Ziyao
%A Tian, Bowei
%A Liu, Meng
%A Shen, Zheyu
%A He, Shwai
%A He, Yexiao
%A Ye, Wanghao
%A Wang, Yiting
%A Li, Ang
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F sun-etal-2026-coin
%X As post-training techniques evolve, large language models (LLMs) are increasingly augmented with structured multi-step reasoning abilities, often optimized through reinforcement learning. These reasoning-enhanced models outperform standard LLMs on complex tasks and now underpin many commercial LLM APIs. However, to protect proprietary behavior and reduce verbosity, providers typically conceal the reasoning traces while returning only the final answer. This opacity introduces a critical transparency gap: users are billed for invisible reasoning tokens, which often account for the majority of the cost, yet have no means to verify their authenticity. This opens the door to token count manipulation, where providers may overreport token usage or inject synthetic, low-effort tokens to inflate charges, a threat that can be carried out at near-zero cost. To address this issue, we propose CoIn, a verification framework that audits both the quantity and semantic validity of hidden tokens. CoIn constructs a verifiable hash tree from token embedding fingerprints to check token counts, and uses embedding-based relevance matching to detect fabricated reasoning content. Experiments demonstrate that CoIn, when deployed as a trusted third-party auditor, can effectively detect token count manipulation with a success rate reaching up to 94.7%, demonstrating a strong ability to restore billing transparency in opaque LLM services. The dataset and code are available at https://github.com/CASE-Lab-UMD/LLM-Auditing-CoIn .
%U https://aclanthology.org/2026.findings-emnlp.363/
%P 7594-7620
Markdown (Informal)
[CoIn: Counting the Invisible Reasoning Tokens in Commercial Opaque LLM APIs](https://aclanthology.org/2026.findings-emnlp.363/) (Sun et al., Findings 2026)
ACL
- Guoheng Sun, Ziyao Wang, Bowei Tian, Meng Liu, Zheyu Shen, Shwai He, Yexiao He, Wanghao Ye, Yiting Wang, and Ang Li. 2026. CoIn: Counting the Invisible Reasoning Tokens in Commercial Opaque LLM APIs. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 7594–7620, Budapest, Hungary. Association for Computational Linguistics.