@inproceedings{park-etal-2026-quantization,
title = "Quantization Breaks the Format: How Weight Compression Undermines Chat Format Safety in On-Device {LLM}",
author = "Park, JuneYoung and
Kim, Seongwan and
Lee, Jaeho",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.37/",
pages = "626--640",
ISBN = "979-8-89176-460-6",
abstract = "On-device LLM deployment routinely serves GGUF-quantized models whose safety is ver- ified at full precision rather than on the llama.cpp/GGUF stack end users actually run. We stress-test nine production mod- els across four GGUF levels against seven CHATBUG chat-format attacks on 520 Ad- vBench instructions, served through the same llama.cpp binary that powers Ollama, and report three deployment-relevant findings. (1) Q2{\_}K vulnerability concentrates in the smallest on-device models: Llama-3.2-1B reaches 75.3{\%} ASR-R at Q2{\_}K (+26.9 pp vs. Q8{\_}0) while Gemma-3-4B is essentially un- affected (+3.0 pp), so model choice dominates quantization-level choice within the on-device regime. (2) The vulnerability is a property of the deployed system: a subset reproduces end- to-end through Ollama (median |{\ensuremath{\Delta}}|= 2.7 pp). (3) Keyword- and moderator-based metrics diverge in magnitude: Llama-2 Mismatch-{\ensuremath{\varnothing}} shifts ASR-R +55.8 pp but ASR-M{\ensuremath{-}}1.7 pp at Q2{\_}K, while Llama-2 Overflow-FS is con- firmed unsafe by both (+16.8 pp ASR-M). We release a one-page deployment-safety checklist and GGUF-native diagnostic probes, and iden- tify single-layer probes as insufficient cross- model predictors of ASR change, motivating per-layer llama.cpp instrumentation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="park-etal-2026-quantization">
<titleInfo>
<title>Quantization Breaks the Format: How Weight Compression Undermines Chat Format Safety in On-Device LLM</title>
</titleInfo>
<name type="personal">
<namePart type="given">JuneYoung</namePart>
<namePart type="family">Park</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Seongwan</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaeho</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>On-device LLM deployment routinely serves GGUF-quantized models whose safety is ver- ified at full precision rather than on the llama.cpp/GGUF stack end users actually run. We stress-test nine production mod- els across four GGUF levels against seven CHATBUG chat-format attacks on 520 Ad- vBench instructions, served through the same llama.cpp binary that powers Ollama, and report three deployment-relevant findings. (1) Q2_K vulnerability concentrates in the smallest on-device models: Llama-3.2-1B reaches 75.3% ASR-R at Q2_K (+26.9 pp vs. Q8_0) while Gemma-3-4B is essentially un- affected (+3.0 pp), so model choice dominates quantization-level choice within the on-device regime. (2) The vulnerability is a property of the deployed system: a subset reproduces end- to-end through Ollama (median |\ensuremathΔ|= 2.7 pp). (3) Keyword- and moderator-based metrics diverge in magnitude: Llama-2 Mismatch-\ensuremathǎrnothing shifts ASR-R +55.8 pp but ASR-M\ensuremath-1.7 pp at Q2_K, while Llama-2 Overflow-FS is con- firmed unsafe by both (+16.8 pp ASR-M). We release a one-page deployment-safety checklist and GGUF-native diagnostic probes, and iden- tify single-layer probes as insufficient cross- model predictors of ASR change, motivating per-layer llama.cpp instrumentation.</abstract>
<identifier type="citekey">park-etal-2026-quantization</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.37/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>626</start>
<end>640</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Quantization Breaks the Format: How Weight Compression Undermines Chat Format Safety in On-Device LLM
%A Park, JuneYoung
%A Kim, Seongwan
%A Lee, Jaeho
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F park-etal-2026-quantization
%X On-device LLM deployment routinely serves GGUF-quantized models whose safety is ver- ified at full precision rather than on the llama.cpp/GGUF stack end users actually run. We stress-test nine production mod- els across four GGUF levels against seven CHATBUG chat-format attacks on 520 Ad- vBench instructions, served through the same llama.cpp binary that powers Ollama, and report three deployment-relevant findings. (1) Q2_K vulnerability concentrates in the smallest on-device models: Llama-3.2-1B reaches 75.3% ASR-R at Q2_K (+26.9 pp vs. Q8_0) while Gemma-3-4B is essentially un- affected (+3.0 pp), so model choice dominates quantization-level choice within the on-device regime. (2) The vulnerability is a property of the deployed system: a subset reproduces end- to-end through Ollama (median |\ensuremathΔ|= 2.7 pp). (3) Keyword- and moderator-based metrics diverge in magnitude: Llama-2 Mismatch-\ensuremathǎrnothing shifts ASR-R +55.8 pp but ASR-M\ensuremath-1.7 pp at Q2_K, while Llama-2 Overflow-FS is con- firmed unsafe by both (+16.8 pp ASR-M). We release a one-page deployment-safety checklist and GGUF-native diagnostic probes, and iden- tify single-layer probes as insufficient cross- model predictors of ASR change, motivating per-layer llama.cpp instrumentation.
%U https://aclanthology.org/2026.emnlp-industry.37/
%P 626-640
Markdown (Informal)
[Quantization Breaks the Format: How Weight Compression Undermines Chat Format Safety in On-Device LLM](https://aclanthology.org/2026.emnlp-industry.37/) (Park et al., EMNLP 2026)
ACL