@inproceedings{wang-etal-2026-beyond-dense,
title = "Beyond Dense States: Sparse Transcoders as Causally Testable Operators for {LLM} Latent Reasoning",
author = "Wang, Yadong and
Chen, Haodong and
Tian, Yu and
Geng, Chuanxing and
Liang, Dong and
Chen, Xiang",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1027/",
pages = "23616--23629",
ISBN = "979-8-89176-459-0",
abstract = "Latent reasoning reduces the token-generation cost of chain-of-thought reasoning by replacing explicit intermediate tokens with continuous latent transitions. However, existing latent reasoning methods usually rely on dense and entangled transitions, making their reasoning trajectories difficult to inspect or intervene on. We introduce LSTR (Latent Sparse Transcoder Reasoning), a framework that turns sparse transcoders from post-hoc diagnostic tools into in-loop, intervenable transition components for latent reasoning. At each latent step, a Latent Transition Transcoder (LTT) combines a linear skip path with a Top-k sparse innovation path, exposing a small set of active sparse features. Under matched compression settings, LSTR offers a mechanistically inspectable alternative to dense latent reasoning. On GSM8K-Aug, ablating only a few top-active sparse features reduces accuracy by up to 16.5{\%}, whereas analogous interventions have much smaller effects in dense latent baselines. These results indicate that the active sparse features are causally involved in the latent transition process, rather than merely post-hoc descriptors. Additional experiments on mathematical benchmarks and StrategyQA suggest that sparse latent transitions can preserve the compression benefits of latent reasoning while making the resulting trajectories more inspectable and intervenable."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-beyond-dense">
<titleInfo>
<title>Beyond Dense States: Sparse Transcoders as Causally Testable Operators for LLM Latent Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yadong</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haodong</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yu</namePart>
<namePart type="family">Tian</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chuanxing</namePart>
<namePart type="family">Geng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dong</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiang</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Latent reasoning reduces the token-generation cost of chain-of-thought reasoning by replacing explicit intermediate tokens with continuous latent transitions. However, existing latent reasoning methods usually rely on dense and entangled transitions, making their reasoning trajectories difficult to inspect or intervene on. We introduce LSTR (Latent Sparse Transcoder Reasoning), a framework that turns sparse transcoders from post-hoc diagnostic tools into in-loop, intervenable transition components for latent reasoning. At each latent step, a Latent Transition Transcoder (LTT) combines a linear skip path with a Top-k sparse innovation path, exposing a small set of active sparse features. Under matched compression settings, LSTR offers a mechanistically inspectable alternative to dense latent reasoning. On GSM8K-Aug, ablating only a few top-active sparse features reduces accuracy by up to 16.5%, whereas analogous interventions have much smaller effects in dense latent baselines. These results indicate that the active sparse features are causally involved in the latent transition process, rather than merely post-hoc descriptors. Additional experiments on mathematical benchmarks and StrategyQA suggest that sparse latent transitions can preserve the compression benefits of latent reasoning while making the resulting trajectories more inspectable and intervenable.</abstract>
<identifier type="citekey">wang-etal-2026-beyond-dense</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1027/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>23616</start>
<end>23629</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Dense States: Sparse Transcoders as Causally Testable Operators for LLM Latent Reasoning
%A Wang, Yadong
%A Chen, Haodong
%A Tian, Yu
%A Geng, Chuanxing
%A Liang, Dong
%A Chen, Xiang
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F wang-etal-2026-beyond-dense
%X Latent reasoning reduces the token-generation cost of chain-of-thought reasoning by replacing explicit intermediate tokens with continuous latent transitions. However, existing latent reasoning methods usually rely on dense and entangled transitions, making their reasoning trajectories difficult to inspect or intervene on. We introduce LSTR (Latent Sparse Transcoder Reasoning), a framework that turns sparse transcoders from post-hoc diagnostic tools into in-loop, intervenable transition components for latent reasoning. At each latent step, a Latent Transition Transcoder (LTT) combines a linear skip path with a Top-k sparse innovation path, exposing a small set of active sparse features. Under matched compression settings, LSTR offers a mechanistically inspectable alternative to dense latent reasoning. On GSM8K-Aug, ablating only a few top-active sparse features reduces accuracy by up to 16.5%, whereas analogous interventions have much smaller effects in dense latent baselines. These results indicate that the active sparse features are causally involved in the latent transition process, rather than merely post-hoc descriptors. Additional experiments on mathematical benchmarks and StrategyQA suggest that sparse latent transitions can preserve the compression benefits of latent reasoning while making the resulting trajectories more inspectable and intervenable.
%U https://aclanthology.org/2026.emnlp-main.1027/
%P 23616-23629
Markdown (Informal)
[Beyond Dense States: Sparse Transcoders as Causally Testable Operators for LLM Latent Reasoning](https://aclanthology.org/2026.emnlp-main.1027/) (Wang et al., EMNLP 2026)
ACL