@inproceedings{li-etal-2026-complexity-clarity,
title = "From Complexity to Clarity: A Systematic Study of Geometric Dynamics in Hidden Representations of {LLM}s",
author = "Li, Xiang and
Sun, Luzhe and
Walter, Matthew and
Zhou, Jiawei",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1221/",
pages = "25843--25861",
ISBN = "979-8-89176-462-0",
abstract = "Large language models (LLMs) perform inference through hidden representations that evolve across layers and generated tokens, forming complex trajectories in hidden representation space. We study the geometric dynamics of these trajectories for label-free model self-evaluation problem: does the hidden-state tensor already contain model response correctness signals, and where do those signals reside? Our analysis is organized along three intrinsic dimensions of inference-time computation: layer depth, token span, and representation space. Across seven benchmarks spanning mathematical reasoning, commonsense reasoning, and multilingual understanding, and two primary instruction-tuned LLMs, we find that correctness separability is highly localized, consistently concentrating in late-layer representations and limited token regions. Motivated by this, we propose Norm Drop Rate (NDR), a minimal diagnostic metric based on final-layer norm contraction. NDR achieves competitive AUROC across tested models and the best AUPR on Qwen2-7B-Instruct, while remaining the simplest in design and most robust when controlled for response length or changing representation bases. These results suggest that hidden-state trajectory analysis should focus less on increasingly complex diagnoses but more on identifying simple, localized signals."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-etal-2026-complexity-clarity">
<titleInfo>
<title>From Complexity to Clarity: A Systematic Study of Geometric Dynamics in Hidden Representations of LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xiang</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Luzhe</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Matthew</namePart>
<namePart type="family">Walter</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiawei</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large language models (LLMs) perform inference through hidden representations that evolve across layers and generated tokens, forming complex trajectories in hidden representation space. We study the geometric dynamics of these trajectories for label-free model self-evaluation problem: does the hidden-state tensor already contain model response correctness signals, and where do those signals reside? Our analysis is organized along three intrinsic dimensions of inference-time computation: layer depth, token span, and representation space. Across seven benchmarks spanning mathematical reasoning, commonsense reasoning, and multilingual understanding, and two primary instruction-tuned LLMs, we find that correctness separability is highly localized, consistently concentrating in late-layer representations and limited token regions. Motivated by this, we propose Norm Drop Rate (NDR), a minimal diagnostic metric based on final-layer norm contraction. NDR achieves competitive AUROC across tested models and the best AUPR on Qwen2-7B-Instruct, while remaining the simplest in design and most robust when controlled for response length or changing representation bases. These results suggest that hidden-state trajectory analysis should focus less on increasingly complex diagnoses but more on identifying simple, localized signals.</abstract>
<identifier type="citekey">li-etal-2026-complexity-clarity</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1221/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>25843</start>
<end>25861</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T From Complexity to Clarity: A Systematic Study of Geometric Dynamics in Hidden Representations of LLMs
%A Li, Xiang
%A Sun, Luzhe
%A Walter, Matthew
%A Zhou, Jiawei
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F li-etal-2026-complexity-clarity
%X Large language models (LLMs) perform inference through hidden representations that evolve across layers and generated tokens, forming complex trajectories in hidden representation space. We study the geometric dynamics of these trajectories for label-free model self-evaluation problem: does the hidden-state tensor already contain model response correctness signals, and where do those signals reside? Our analysis is organized along three intrinsic dimensions of inference-time computation: layer depth, token span, and representation space. Across seven benchmarks spanning mathematical reasoning, commonsense reasoning, and multilingual understanding, and two primary instruction-tuned LLMs, we find that correctness separability is highly localized, consistently concentrating in late-layer representations and limited token regions. Motivated by this, we propose Norm Drop Rate (NDR), a minimal diagnostic metric based on final-layer norm contraction. NDR achieves competitive AUROC across tested models and the best AUPR on Qwen2-7B-Instruct, while remaining the simplest in design and most robust when controlled for response length or changing representation bases. These results suggest that hidden-state trajectory analysis should focus less on increasingly complex diagnoses but more on identifying simple, localized signals.
%U https://aclanthology.org/2026.findings-emnlp.1221/
%P 25843-25861
Markdown (Informal)
[From Complexity to Clarity: A Systematic Study of Geometric Dynamics in Hidden Representations of LLMs](https://aclanthology.org/2026.findings-emnlp.1221/) (Li et al., Findings 2026)
ACL