@inproceedings{gao-etal-2026-soft,
title = "Soft Tokens Are Off-Manifold: Geometric Correction for Continuous-Space Reasoning in Large Language Models",
author = "Gao, Jing and
Jin, Weiqiang and
Mi, Yunqi and
Ye, Peigen and
Du, Hongyang and
Zhao, Biao and
Deng, Yimin and
Liu, Chengxu and
Qian, Xueming and
Zhao, Guoshuai",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.273/",
pages = "6198--6223",
ISBN = "979-8-89176-459-0",
abstract = "Continuous-space reasoning methods replace discrete intermediate tokens with probability-weighted soft embeddings, aiming to preserve semantic information that greedy Chain-of-Thought decoding discards. We show that this promise is limited by local representation geometry and the latent semantic manifold. When the next-token distribution is low-entropy and concentrated, a soft token becomes nearly collinear with the Top-1 embedding and collapses toward greedy decoding. When the distribution is high-entropy, diffuse, or stochastically perturbed, the continuous state may leave the greedy path but drift away from the local scale of naturally occurring candidate representations. We diagnose these two regimes with step-level probes over representation divergence, local norm calibration, and behavior-cloud membership. To address them, we propose Manifold Thinking, a training-free inference framework that combines structured tangent movement away from the Top-1 direction with local norm-calibrated projection back to the candidate-token neighborhood. Across mathematical and programming benchmarks, Manifold Thinking improves over discrete CoT baselines and baselines covering the latest stochastic Soft Thinking strategies, suggesting that useful continuous reasoning requires geometric correction rather than unconstrained linear token mixing alone."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="gao-etal-2026-soft">
<titleInfo>
<title>Soft Tokens Are Off-Manifold: Geometric Correction for Continuous-Space Reasoning in Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jing</namePart>
<namePart type="family">Gao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weiqiang</namePart>
<namePart type="family">Jin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yunqi</namePart>
<namePart type="family">Mi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peigen</namePart>
<namePart type="family">Ye</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hongyang</namePart>
<namePart type="family">Du</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Biao</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yimin</namePart>
<namePart type="family">Deng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chengxu</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xueming</namePart>
<namePart type="family">Qian</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Guoshuai</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Continuous-space reasoning methods replace discrete intermediate tokens with probability-weighted soft embeddings, aiming to preserve semantic information that greedy Chain-of-Thought decoding discards. We show that this promise is limited by local representation geometry and the latent semantic manifold. When the next-token distribution is low-entropy and concentrated, a soft token becomes nearly collinear with the Top-1 embedding and collapses toward greedy decoding. When the distribution is high-entropy, diffuse, or stochastically perturbed, the continuous state may leave the greedy path but drift away from the local scale of naturally occurring candidate representations. We diagnose these two regimes with step-level probes over representation divergence, local norm calibration, and behavior-cloud membership. To address them, we propose Manifold Thinking, a training-free inference framework that combines structured tangent movement away from the Top-1 direction with local norm-calibrated projection back to the candidate-token neighborhood. Across mathematical and programming benchmarks, Manifold Thinking improves over discrete CoT baselines and baselines covering the latest stochastic Soft Thinking strategies, suggesting that useful continuous reasoning requires geometric correction rather than unconstrained linear token mixing alone.</abstract>
<identifier type="citekey">gao-etal-2026-soft</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.273/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>6198</start>
<end>6223</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Soft Tokens Are Off-Manifold: Geometric Correction for Continuous-Space Reasoning in Large Language Models
%A Gao, Jing
%A Jin, Weiqiang
%A Mi, Yunqi
%A Ye, Peigen
%A Du, Hongyang
%A Zhao, Biao
%A Deng, Yimin
%A Liu, Chengxu
%A Qian, Xueming
%A Zhao, Guoshuai
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F gao-etal-2026-soft
%X Continuous-space reasoning methods replace discrete intermediate tokens with probability-weighted soft embeddings, aiming to preserve semantic information that greedy Chain-of-Thought decoding discards. We show that this promise is limited by local representation geometry and the latent semantic manifold. When the next-token distribution is low-entropy and concentrated, a soft token becomes nearly collinear with the Top-1 embedding and collapses toward greedy decoding. When the distribution is high-entropy, diffuse, or stochastically perturbed, the continuous state may leave the greedy path but drift away from the local scale of naturally occurring candidate representations. We diagnose these two regimes with step-level probes over representation divergence, local norm calibration, and behavior-cloud membership. To address them, we propose Manifold Thinking, a training-free inference framework that combines structured tangent movement away from the Top-1 direction with local norm-calibrated projection back to the candidate-token neighborhood. Across mathematical and programming benchmarks, Manifold Thinking improves over discrete CoT baselines and baselines covering the latest stochastic Soft Thinking strategies, suggesting that useful continuous reasoning requires geometric correction rather than unconstrained linear token mixing alone.
%U https://aclanthology.org/2026.emnlp-main.273/
%P 6198-6223
Markdown (Informal)
[Soft Tokens Are Off-Manifold: Geometric Correction for Continuous-Space Reasoning in Large Language Models](https://aclanthology.org/2026.emnlp-main.273/) (Gao et al., EMNLP 2026)
ACL
- Jing Gao, Weiqiang Jin, Yunqi Mi, Peigen Ye, Hongyang Du, Biao Zhao, Yimin Deng, Chengxu Liu, Xueming Qian, and Guoshuai Zhao. 2026. Soft Tokens Are Off-Manifold: Geometric Correction for Continuous-Space Reasoning in Large Language Models. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 6198–6223, Budapest, Hungary. Association for Computational Linguistics.