@inproceedings{liu-etal-2026-layer,
title = "Layer-Order Inversion: Rethinking Latent Multi-Hop Reasoning in Large Language Models",
author = "Liu, Xukai and
Liu, Ye and
Zhang, Jipeng and
Zhang, Yanghai and
Zhang, Kai and
Liu, Qi",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1802/",
pages = "41041--41058",
ISBN = "979-8-89176-459-0",
abstract = "Large language models (LLMs) perform well on multi-hop reasoning, yet how they internally compose multiple facts remains unclear. Recent work proposes **hop-aligned circuit hypothesis**, suggesting that bridge entities are computed sequentially across layers before later-hop answers. Through systematic analyses on real-world multi-hop queries, we show that this hop-aligned assumption does not generalize: later-hop answer entities can become decodable earlier than bridge entities, a phenomenon we call **layer-order inversion**, which strengthens with total hops. To explain this behavior, we propose a **probabilistic recall-and-extract** framework that models multi-hop reasoning as broad probabilistic recall in shallow MLP layers followed by selective extraction in deeper attention layers. Through probing and causal intervention analyses, we further show that shallow recall of deeper-hop knowledge can act as an intuition-like signal for answer selection, with stronger effects in weaker models and higher-hop queries. Our framework further explains prior empirical phenomena and provides a mechanistic diagnosis of multi-hop failures despite correct single-hop knowledge. Code is available at https://anonymous.4open.science/r/Layer-Order-Inversion/."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-layer">
<titleInfo>
<title>Layer-Order Inversion: Rethinking Latent Multi-Hop Reasoning in Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xukai</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ye</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jipeng</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yanghai</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kai</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qi</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large language models (LLMs) perform well on multi-hop reasoning, yet how they internally compose multiple facts remains unclear. Recent work proposes **hop-aligned circuit hypothesis**, suggesting that bridge entities are computed sequentially across layers before later-hop answers. Through systematic analyses on real-world multi-hop queries, we show that this hop-aligned assumption does not generalize: later-hop answer entities can become decodable earlier than bridge entities, a phenomenon we call **layer-order inversion**, which strengthens with total hops. To explain this behavior, we propose a **probabilistic recall-and-extract** framework that models multi-hop reasoning as broad probabilistic recall in shallow MLP layers followed by selective extraction in deeper attention layers. Through probing and causal intervention analyses, we further show that shallow recall of deeper-hop knowledge can act as an intuition-like signal for answer selection, with stronger effects in weaker models and higher-hop queries. Our framework further explains prior empirical phenomena and provides a mechanistic diagnosis of multi-hop failures despite correct single-hop knowledge. Code is available at https://anonymous.4open.science/r/Layer-Order-Inversion/.</abstract>
<identifier type="citekey">liu-etal-2026-layer</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1802/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>41041</start>
<end>41058</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Layer-Order Inversion: Rethinking Latent Multi-Hop Reasoning in Large Language Models
%A Liu, Xukai
%A Liu, Ye
%A Zhang, Jipeng
%A Zhang, Yanghai
%A Zhang, Kai
%A Liu, Qi
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F liu-etal-2026-layer
%X Large language models (LLMs) perform well on multi-hop reasoning, yet how they internally compose multiple facts remains unclear. Recent work proposes **hop-aligned circuit hypothesis**, suggesting that bridge entities are computed sequentially across layers before later-hop answers. Through systematic analyses on real-world multi-hop queries, we show that this hop-aligned assumption does not generalize: later-hop answer entities can become decodable earlier than bridge entities, a phenomenon we call **layer-order inversion**, which strengthens with total hops. To explain this behavior, we propose a **probabilistic recall-and-extract** framework that models multi-hop reasoning as broad probabilistic recall in shallow MLP layers followed by selective extraction in deeper attention layers. Through probing and causal intervention analyses, we further show that shallow recall of deeper-hop knowledge can act as an intuition-like signal for answer selection, with stronger effects in weaker models and higher-hop queries. Our framework further explains prior empirical phenomena and provides a mechanistic diagnosis of multi-hop failures despite correct single-hop knowledge. Code is available at https://anonymous.4open.science/r/Layer-Order-Inversion/.
%U https://aclanthology.org/2026.emnlp-main.1802/
%P 41041-41058
Markdown (Informal)
[Layer-Order Inversion: Rethinking Latent Multi-Hop Reasoning in Large Language Models](https://aclanthology.org/2026.emnlp-main.1802/) (Liu et al., EMNLP 2026)
ACL