@inproceedings{purwarianti-etal-2026-skip,
title = "Skip One Layer? A Multilingual and Multi-Task Study of Single-Layer Skipping in Modern Language Models",
author = "Purwarianti, Ayu and
Krishna, Antonio Natthan and
Firdaus, Muhammad Risqi and
Zhafran, Hanif Muhammad and
Tansil, Wilson and
Kistijantoro, Achmad Imam and
Ahmadi, Nur and
Mulyawan, Rahmat and
Lestari, Dessi Puji and
Kielian, Gregory",
editor = "Wang, Yimu and
Choi, Yee Man and
Wu, Di and
Ouyang, Siqi and
Azadani, Mozhgan Nasr and
Fung, Yi R. (May)",
booktitle = "Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently ({G}round{LM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.groundlm-1.49/",
pages = "649--665",
ISBN = "979-8-89176-478-1",
abstract = "Modern language models are typically deployed using all transformer layers during inference, implicitly assuming that every layer contributes positively to downstream performance. Prior studies on layer skipping often focus on limited settings and commonly associate removable layers with specific transformer regions, particularly middle layers. However, whether single-layer skipping generalizes across diverse tasks, languages, and model architectures remains underexplored.\par In this work, we conduct a multilingual and multi-task empirical study of single-layer skipping in modern instruct-tuned language models. We evaluate 9 text-only models spanning multiple model families and parameter scales, including Gemma 3, Llama 3.2, and Qwen 3, OLMo 2 without modifying model parameters. Our experiments systematically evaluate single-layer skipping across all layers on diverse benchmarks covering NLU, reasoning, bias evaluation, code generation, instruction following, and multilingual knowledge evaluation, including MMLU across 57 subject domains and 10 typologically diverse languages.\par Our results show that the effect of single-layer skipping is highly context-dependent. We show that skipping a single transformer layer is an effective strategy for leveraging the capabilities of a language model: there is a high chance of achieving better benchmark performance while simultaneously reducing computation during inference. However, the optimal skipped layer varies depending on model architecture, task, domain, and language. Beneficial skipped layers occur most frequently in the second half of the network, though they also emerge in early layers in some cases. We further observe that performance improvements are more common in structured prediction tasks, whereas degradations more frequently occur in tasks requiring precise generation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="purwarianti-etal-2026-skip">
<titleInfo>
<title>Skip One Layer? A Multilingual and Multi-Task Study of Single-Layer Skipping in Modern Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Antonio</namePart>
<namePart type="given">Natthan</namePart>
<namePart type="family">Krishna</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Muhammad</namePart>
<namePart type="given">Risqi</namePart>
<namePart type="family">Firdaus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hanif</namePart>
<namePart type="given">Muhammad</namePart>
<namePart type="family">Zhafran</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wilson</namePart>
<namePart type="family">Tansil</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Achmad</namePart>
<namePart type="given">Imam</namePart>
<namePart type="family">Kistijantoro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nur</namePart>
<namePart type="family">Ahmadi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rahmat</namePart>
<namePart type="family">Mulyawan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dessi</namePart>
<namePart type="given">Puji</namePart>
<namePart type="family">Lestari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gregory</namePart>
<namePart type="family">Kielian</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yee</namePart>
<namePart type="given">Man</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siqi</namePart>
<namePart type="family">Ouyang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mozhgan</namePart>
<namePart type="given">Nasr</namePart>
<namePart type="family">Azadani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="given">R</namePart>
<namePart type="given">(May)</namePart>
<namePart type="family">Fung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-478-1</identifier>
</relatedItem>
<abstract>Modern language models are typically deployed using all transformer layers during inference, implicitly assuming that every layer contributes positively to downstream performance. Prior studies on layer skipping often focus on limited settings and commonly associate removable layers with specific transformer regions, particularly middle layers. However, whether single-layer skipping generalizes across diverse tasks, languages, and model architectures remains underexplored.\par In this work, we conduct a multilingual and multi-task empirical study of single-layer skipping in modern instruct-tuned language models. We evaluate 9 text-only models spanning multiple model families and parameter scales, including Gemma 3, Llama 3.2, and Qwen 3, OLMo 2 without modifying model parameters. Our experiments systematically evaluate single-layer skipping across all layers on diverse benchmarks covering NLU, reasoning, bias evaluation, code generation, instruction following, and multilingual knowledge evaluation, including MMLU across 57 subject domains and 10 typologically diverse languages.\par Our results show that the effect of single-layer skipping is highly context-dependent. We show that skipping a single transformer layer is an effective strategy for leveraging the capabilities of a language model: there is a high chance of achieving better benchmark performance while simultaneously reducing computation during inference. However, the optimal skipped layer varies depending on model architecture, task, domain, and language. Beneficial skipped layers occur most frequently in the second half of the network, though they also emerge in early layers in some cases. We further observe that performance improvements are more common in structured prediction tasks, whereas degradations more frequently occur in tasks requiring precise generation.</abstract>
<identifier type="citekey">purwarianti-etal-2026-skip</identifier>
<location>
<url>https://aclanthology.org/2026.groundlm-1.49/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>649</start>
<end>665</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Skip One Layer? A Multilingual and Multi-Task Study of Single-Layer Skipping in Modern Language Models
%A Purwarianti, Ayu
%A Krishna, Antonio Natthan
%A Firdaus, Muhammad Risqi
%A Zhafran, Hanif Muhammad
%A Tansil, Wilson
%A Kistijantoro, Achmad Imam
%A Ahmadi, Nur
%A Mulyawan, Rahmat
%A Lestari, Dessi Puji
%A Kielian, Gregory
%Y Wang, Yimu
%Y Choi, Yee Man
%Y Wu, Di
%Y Ouyang, Siqi
%Y Azadani, Mozhgan Nasr
%Y Fung, Yi R. (May)
%S Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-478-1
%F purwarianti-etal-2026-skip
%X Modern language models are typically deployed using all transformer layers during inference, implicitly assuming that every layer contributes positively to downstream performance. Prior studies on layer skipping often focus on limited settings and commonly associate removable layers with specific transformer regions, particularly middle layers. However, whether single-layer skipping generalizes across diverse tasks, languages, and model architectures remains underexplored.\par In this work, we conduct a multilingual and multi-task empirical study of single-layer skipping in modern instruct-tuned language models. We evaluate 9 text-only models spanning multiple model families and parameter scales, including Gemma 3, Llama 3.2, and Qwen 3, OLMo 2 without modifying model parameters. Our experiments systematically evaluate single-layer skipping across all layers on diverse benchmarks covering NLU, reasoning, bias evaluation, code generation, instruction following, and multilingual knowledge evaluation, including MMLU across 57 subject domains and 10 typologically diverse languages.\par Our results show that the effect of single-layer skipping is highly context-dependent. We show that skipping a single transformer layer is an effective strategy for leveraging the capabilities of a language model: there is a high chance of achieving better benchmark performance while simultaneously reducing computation during inference. However, the optimal skipped layer varies depending on model architecture, task, domain, and language. Beneficial skipped layers occur most frequently in the second half of the network, though they also emerge in early layers in some cases. We further observe that performance improvements are more common in structured prediction tasks, whereas degradations more frequently occur in tasks requiring precise generation.
%U https://aclanthology.org/2026.groundlm-1.49/
%P 649-665
Markdown (Informal)
[Skip One Layer? A Multilingual and Multi-Task Study of Single-Layer Skipping in Modern Language Models](https://aclanthology.org/2026.groundlm-1.49/) (Purwarianti et al., GroundLM 2026)
ACL
- Ayu Purwarianti, Antonio Natthan Krishna, Muhammad Risqi Firdaus, Hanif Muhammad Zhafran, Wilson Tansil, Achmad Imam Kistijantoro, Nur Ahmadi, Rahmat Mulyawan, Dessi Puji Lestari, and Gregory Kielian. 2026. Skip One Layer? A Multilingual and Multi-Task Study of Single-Layer Skipping in Modern Language Models. In Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026), pages 649–665, Budapest, Hungary. Association for Computational Linguistics.