@inproceedings{retsinas-etal-2026-breaking,
title = "Breaking Causality in {LLM} Training with Sparsity and Quantization",
author = "Retsinas, George and
Agrusa, Anjulie and
Rybakov, Oleg and
Knight, Nicholas and
Khona, Mikail and
Yu, Chong and
Blankevoort, Tijmen and
Pool, Jeff",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2432/",
pages = "55202--55221",
ISBN = "979-8-89176-459-0",
abstract = "Autoregressive training requires that the computation at each position ignore future tokens. Causal masking enforces this for self-attention, but efficient training recipes can still break causality when they compute shared statistics across the sequence dimension. We study two such operations: structured activation sparsity and block-scaled activation quantization. Applying these operators along the sequence dimension is attractive for hardware efficiency and can look harmless, but it can break causality when the mask or scale applied to a past token depends on future activations. We show that models do exploit this dependency: the training objective continues to improve under the non-causal operator, but causal validation degrades when the future-dependent computation is removed. We analyze why mitigations that keep the operator in the forward pass can postpone or weaken exploitation, but cannot remove the future-to-past dependency. The solution is to remove non-causal statistics from the forward computation used for prediction, and we show that existing sparsity and quantization recipes do respect this rule, though causality was not an explicit design decision."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="retsinas-etal-2026-breaking">
<titleInfo>
<title>Breaking Causality in LLM Training with Sparsity and Quantization</title>
</titleInfo>
<name type="personal">
<namePart type="given">George</namePart>
<namePart type="family">Retsinas</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anjulie</namePart>
<namePart type="family">Agrusa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Oleg</namePart>
<namePart type="family">Rybakov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicholas</namePart>
<namePart type="family">Knight</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mikail</namePart>
<namePart type="family">Khona</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chong</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tijmen</namePart>
<namePart type="family">Blankevoort</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jeff</namePart>
<namePart type="family">Pool</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Autoregressive training requires that the computation at each position ignore future tokens. Causal masking enforces this for self-attention, but efficient training recipes can still break causality when they compute shared statistics across the sequence dimension. We study two such operations: structured activation sparsity and block-scaled activation quantization. Applying these operators along the sequence dimension is attractive for hardware efficiency and can look harmless, but it can break causality when the mask or scale applied to a past token depends on future activations. We show that models do exploit this dependency: the training objective continues to improve under the non-causal operator, but causal validation degrades when the future-dependent computation is removed. We analyze why mitigations that keep the operator in the forward pass can postpone or weaken exploitation, but cannot remove the future-to-past dependency. The solution is to remove non-causal statistics from the forward computation used for prediction, and we show that existing sparsity and quantization recipes do respect this rule, though causality was not an explicit design decision.</abstract>
<identifier type="citekey">retsinas-etal-2026-breaking</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2432/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>55202</start>
<end>55221</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Breaking Causality in LLM Training with Sparsity and Quantization
%A Retsinas, George
%A Agrusa, Anjulie
%A Rybakov, Oleg
%A Knight, Nicholas
%A Khona, Mikail
%A Yu, Chong
%A Blankevoort, Tijmen
%A Pool, Jeff
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F retsinas-etal-2026-breaking
%X Autoregressive training requires that the computation at each position ignore future tokens. Causal masking enforces this for self-attention, but efficient training recipes can still break causality when they compute shared statistics across the sequence dimension. We study two such operations: structured activation sparsity and block-scaled activation quantization. Applying these operators along the sequence dimension is attractive for hardware efficiency and can look harmless, but it can break causality when the mask or scale applied to a past token depends on future activations. We show that models do exploit this dependency: the training objective continues to improve under the non-causal operator, but causal validation degrades when the future-dependent computation is removed. We analyze why mitigations that keep the operator in the forward pass can postpone or weaken exploitation, but cannot remove the future-to-past dependency. The solution is to remove non-causal statistics from the forward computation used for prediction, and we show that existing sparsity and quantization recipes do respect this rule, though causality was not an explicit design decision.
%U https://aclanthology.org/2026.emnlp-main.2432/
%P 55202-55221
Markdown (Informal)
[Breaking Causality in LLM Training with Sparsity and Quantization](https://aclanthology.org/2026.emnlp-main.2432/) (Retsinas et al., EMNLP 2026)
ACL
- George Retsinas, Anjulie Agrusa, Oleg Rybakov, Nicholas Knight, Mikail Khona, Chong Yu, Tijmen Blankevoort, and Jeff Pool. 2026. Breaking Causality in LLM Training with Sparsity and Quantization. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 55202–55221, Budapest, Hungary. Association for Computational Linguistics.