@inproceedings{zhang-etal-2026-branch,
title = "Branch Later, Compute Less, Think Better: Adaptive Depth Parallelism for Efficient Parallel Prefix-Tuning",
author = "Zhang, Mingqian and
Chen, Mouxiang and
Liu, Zhongxin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1130/",
pages = "25878--25896",
ISBN = "979-8-89176-459-0",
abstract = "Model-internal parallelization improves generation quality by introducing multiple parallel computation paths, but its additional latency and computational overhead hinder practical deployment. Recent studies reveal a functional division across model depth: shallow layers primarily encode input-conditioned features, whereas deeper layers are more involved in reasoning and generation. Since parallel paths process the same input and are optimized toward the same generation objective, their early representations are likely to overlap substantially, suggesting that early-layer parallelism may induce redundant computation. Motivated by this observation, we propose Adaptive Depth Parallelism (ADP), a framework that shares computation in shallow layers and activates parallel branches only after an adaptively selected bifurcation layer. We provide a sufficient-condition analysis for delayed branching and identify an empirical transition in inter-path cosine similarity across depth. This motivates using the stable minimum-similarity layer as a practical estimate of the bifurcation point. Experiments show that ADP achieves 104.83{\%} of baseline performance while using only 39.4{\%} of the computational load."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-branch">
<titleInfo>
<title>Branch Later, Compute Less, Think Better: Adaptive Depth Parallelism for Efficient Parallel Prefix-Tuning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mingqian</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mouxiang</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhongxin</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Model-internal parallelization improves generation quality by introducing multiple parallel computation paths, but its additional latency and computational overhead hinder practical deployment. Recent studies reveal a functional division across model depth: shallow layers primarily encode input-conditioned features, whereas deeper layers are more involved in reasoning and generation. Since parallel paths process the same input and are optimized toward the same generation objective, their early representations are likely to overlap substantially, suggesting that early-layer parallelism may induce redundant computation. Motivated by this observation, we propose Adaptive Depth Parallelism (ADP), a framework that shares computation in shallow layers and activates parallel branches only after an adaptively selected bifurcation layer. We provide a sufficient-condition analysis for delayed branching and identify an empirical transition in inter-path cosine similarity across depth. This motivates using the stable minimum-similarity layer as a practical estimate of the bifurcation point. Experiments show that ADP achieves 104.83% of baseline performance while using only 39.4% of the computational load.</abstract>
<identifier type="citekey">zhang-etal-2026-branch</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1130/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>25878</start>
<end>25896</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Branch Later, Compute Less, Think Better: Adaptive Depth Parallelism for Efficient Parallel Prefix-Tuning
%A Zhang, Mingqian
%A Chen, Mouxiang
%A Liu, Zhongxin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F zhang-etal-2026-branch
%X Model-internal parallelization improves generation quality by introducing multiple parallel computation paths, but its additional latency and computational overhead hinder practical deployment. Recent studies reveal a functional division across model depth: shallow layers primarily encode input-conditioned features, whereas deeper layers are more involved in reasoning and generation. Since parallel paths process the same input and are optimized toward the same generation objective, their early representations are likely to overlap substantially, suggesting that early-layer parallelism may induce redundant computation. Motivated by this observation, we propose Adaptive Depth Parallelism (ADP), a framework that shares computation in shallow layers and activates parallel branches only after an adaptively selected bifurcation layer. We provide a sufficient-condition analysis for delayed branching and identify an empirical transition in inter-path cosine similarity across depth. This motivates using the stable minimum-similarity layer as a practical estimate of the bifurcation point. Experiments show that ADP achieves 104.83% of baseline performance while using only 39.4% of the computational load.
%U https://aclanthology.org/2026.emnlp-main.1130/
%P 25878-25896
Markdown (Informal)
[Branch Later, Compute Less, Think Better: Adaptive Depth Parallelism for Efficient Parallel Prefix-Tuning](https://aclanthology.org/2026.emnlp-main.1130/) (Zhang et al., EMNLP 2026)
ACL