@inproceedings{kennedy-moosavi-2026-initialisation,
title = "Initialisation Determines the Basin: Efficient Codebook Optimisation for Extreme {LLM} Quantization",
author = "Kennedy, Ian W and
Moosavi, Nafise Sadat",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.90/",
pages = "1966--1981",
ISBN = "979-8-89176-439-2",
abstract = "Additive quantization enables extreme LLM compression while preserving lookup-table inference with free-form codebooks. At 2 bits per weight, however, greedy sequential initialisation can cause large quality losses that wider assignment search and PV-tuning do not fully erase. We show that this failure reflects an interaction between representational pressure and sequential codebook coupling. We use $\rho=N/K^M$ as a structural pressure measure and propose OA-EM, a hard generalised-EM initialisation procedure using Hessian-weighted Mahalanobis distance to minimise output-aware reconstruction error. Across four models from 3B to 14B parameters, OA-EM consistently lowers matched post-PV perplexity. Controlled experiments at a fixed bitrate show that $\rho$ alone does not explain sensitivity to initialisation; the number and size of the additive codebooks also matter. A Euclidean-EM ablation and curvature diagnostics show that the gain is not explained by EM refinement alone: OA-EM reduces error in high-curvature directions. On Llama 3.2 3B, OA-EM also improves the measured quality-compute trade-off. Overall, the final quantized solution remains persistently dependent on initial codebook geometry. Code: https://github.com/kenno94-IK/aqlm-oaem."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kennedy-moosavi-2026-initialisation">
<titleInfo>
<title>Initialisation Determines the Basin: Efficient Codebook Optimisation for Extreme LLM Quantization</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ian</namePart>
<namePart type="given">W</namePart>
<namePart type="family">Kennedy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nafise</namePart>
<namePart type="given">Sadat</namePart>
<namePart type="family">Moosavi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Additive quantization enables extreme LLM compression while preserving lookup-table inference with free-form codebooks. At 2 bits per weight, however, greedy sequential initialisation can cause large quality losses that wider assignment search and PV-tuning do not fully erase. We show that this failure reflects an interaction between representational pressure and sequential codebook coupling. We use ρ=N/K^M as a structural pressure measure and propose OA-EM, a hard generalised-EM initialisation procedure using Hessian-weighted Mahalanobis distance to minimise output-aware reconstruction error. Across four models from 3B to 14B parameters, OA-EM consistently lowers matched post-PV perplexity. Controlled experiments at a fixed bitrate show that ρ alone does not explain sensitivity to initialisation; the number and size of the additive codebooks also matter. A Euclidean-EM ablation and curvature diagnostics show that the gain is not explained by EM refinement alone: OA-EM reduces error in high-curvature directions. On Llama 3.2 3B, OA-EM also improves the measured quality-compute trade-off. Overall, the final quantized solution remains persistently dependent on initial codebook geometry. Code: https://github.com/kenno94-IK/aqlm-oaem.</abstract>
<identifier type="citekey">kennedy-moosavi-2026-initialisation</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.90/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>1966</start>
<end>1981</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Initialisation Determines the Basin: Efficient Codebook Optimisation for Extreme LLM Quantization
%A Kennedy, Ian W.
%A Moosavi, Nafise Sadat
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F kennedy-moosavi-2026-initialisation
%X Additive quantization enables extreme LLM compression while preserving lookup-table inference with free-form codebooks. At 2 bits per weight, however, greedy sequential initialisation can cause large quality losses that wider assignment search and PV-tuning do not fully erase. We show that this failure reflects an interaction between representational pressure and sequential codebook coupling. We use ρ=N/K^M as a structural pressure measure and propose OA-EM, a hard generalised-EM initialisation procedure using Hessian-weighted Mahalanobis distance to minimise output-aware reconstruction error. Across four models from 3B to 14B parameters, OA-EM consistently lowers matched post-PV perplexity. Controlled experiments at a fixed bitrate show that ρ alone does not explain sensitivity to initialisation; the number and size of the additive codebooks also matter. A Euclidean-EM ablation and curvature diagnostics show that the gain is not explained by EM refinement alone: OA-EM reduces error in high-curvature directions. On Llama 3.2 3B, OA-EM also improves the measured quality-compute trade-off. Overall, the final quantized solution remains persistently dependent on initial codebook geometry. Code: https://github.com/kenno94-IK/aqlm-oaem.
%U https://aclanthology.org/2026.aacl-long.90/
%P 1966-1981
Markdown (Informal)
[Initialisation Determines the Basin: Efficient Codebook Optimisation for Extreme LLM Quantization](https://aclanthology.org/2026.aacl-long.90/) (Kennedy & Moosavi, AACL-IJCNLP 2026)
ACL