@inproceedings{kim-lee-2026-pretraining,
title = "Pretraining Builds Cooperation, Not Components: Tracking Inter-Layer Coordination via Lie Brackets",
author = "Kim, JaeSeong and
Lee, Suan",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.149/",
pages = "2952--2964",
ISBN = "979-8-89176-444-6",
abstract = "Transformer language models predict the next token by passing through dozens of sequential layers, yet whether adjacent layers operate independently or coordinate their computations{---}and how this relationship forms during pretraining{---}remains largely unknown. We view residual updates of adjacent layers as vector fields and introduce a Lie bracket-based inter-layer coordination metric that quantifies how much two layers depend on each other{'}s outputs. Analyzing pretraining checkpoints of 6 models across 3 architecture families (Pythia, OLMo, Amber), we find that all layer pairs are nearly independent at initialization, but as pretraining progresses, individual layer contributions decrease while coordination systematically increases{---}concentrating in the middle layers that existing metrics (BI, CKA) classify as ``redundant.'' The mechanism is Jacobian amplification: middle layers write small updates but respond sensitively to their neighbors, and this amplification co-emerges with coordination (r {\ensuremath{>}} 0.99). Block duplication experiments confirm that high-coordination blocks are 2.7{--}7.9 more robust than low-coordination blocks, verifying that this cooperative structure substantively contributes to model function. Our findings reframe pretraining as a process that builds cooperative circuits between layers rather than strengthening each layer individually."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kim-lee-2026-pretraining">
<titleInfo>
<title>Pretraining Builds Cooperation, Not Components: Tracking Inter-Layer Coordination via Lie Brackets</title>
</titleInfo>
<name type="personal">
<namePart type="given">JaeSeong</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suan</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>Transformer language models predict the next token by passing through dozens of sequential layers, yet whether adjacent layers operate independently or coordinate their computations—and how this relationship forms during pretraining—remains largely unknown. We view residual updates of adjacent layers as vector fields and introduce a Lie bracket-based inter-layer coordination metric that quantifies how much two layers depend on each other’s outputs. Analyzing pretraining checkpoints of 6 models across 3 architecture families (Pythia, OLMo, Amber), we find that all layer pairs are nearly independent at initialization, but as pretraining progresses, individual layer contributions decrease while coordination systematically increases—concentrating in the middle layers that existing metrics (BI, CKA) classify as “redundant.” The mechanism is Jacobian amplification: middle layers write small updates but respond sensitively to their neighbors, and this amplification co-emerges with coordination (r \ensuremath> 0.99). Block duplication experiments confirm that high-coordination blocks are 2.7–7.9 more robust than low-coordination blocks, verifying that this cooperative structure substantively contributes to model function. Our findings reframe pretraining as a process that builds cooperative circuits between layers rather than strengthening each layer individually.</abstract>
<identifier type="citekey">kim-lee-2026-pretraining</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.149/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>2952</start>
<end>2964</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Pretraining Builds Cooperation, Not Components: Tracking Inter-Layer Coordination via Lie Brackets
%A Kim, JaeSeong
%A Lee, Suan
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F kim-lee-2026-pretraining
%X Transformer language models predict the next token by passing through dozens of sequential layers, yet whether adjacent layers operate independently or coordinate their computations—and how this relationship forms during pretraining—remains largely unknown. We view residual updates of adjacent layers as vector fields and introduce a Lie bracket-based inter-layer coordination metric that quantifies how much two layers depend on each other’s outputs. Analyzing pretraining checkpoints of 6 models across 3 architecture families (Pythia, OLMo, Amber), we find that all layer pairs are nearly independent at initialization, but as pretraining progresses, individual layer contributions decrease while coordination systematically increases—concentrating in the middle layers that existing metrics (BI, CKA) classify as “redundant.” The mechanism is Jacobian amplification: middle layers write small updates but respond sensitively to their neighbors, and this amplification co-emerges with coordination (r \ensuremath> 0.99). Block duplication experiments confirm that high-coordination blocks are 2.7–7.9 more robust than low-coordination blocks, verifying that this cooperative structure substantively contributes to model function. Our findings reframe pretraining as a process that builds cooperative circuits between layers rather than strengthening each layer individually.
%U https://aclanthology.org/2026.findings-aacl.149/
%P 2952-2964
Markdown (Informal)
[Pretraining Builds Cooperation, Not Components: Tracking Inter-Layer Coordination via Lie Brackets](https://aclanthology.org/2026.findings-aacl.149/) (Kim & Lee, Findings 2026)
ACL