@inproceedings{nikolic-etal-2026-displacement,
title = "Displacement Is Not Direction: Evaluating Fidelity Metrics for Quantized {LLM} Deployment",
author = "Nikolic, Milos and
Hadi Zadeh, Ali and
Torres Sanchez, Enrique and
Moshovos, Andreas",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.178/",
pages = "3025--3039",
ISBN = "979-8-89176-460-6",
abstract = "Fidelity metrics such as per-token KL divergence (KLD) against a high-precision reference are often used as low-cost proxies for benchmark quality. We test this practice on 28 Qwen3.6-35B-A3B and 41 Devstral-Small-2-24B quantizations across downstream benchmarks. KLD strongly correlates with benchmark score over the full cohorts ($\rho=-0.72$ on Qwen and $\rho=-0.86$ on Devstral, both with $p<0.001$). However, among quantizations retaining at least 99{\%} of measured BF16 composite quality, KLD provides no dependable ranking signal ($\rho=+0.11$, $p=0.71$, $n=14$ on Qwen; $\rho=+0.02$, $p=0.93$, $n=16$ on Devstral). This collapse persists across measurement variants spanning KLD aggregation, perplexity formulations, top-1 agreement, measurement corpora, and context lengths. At the per-prompt level, KLD provides only weak failure-prediction power and fails as a cross-model router across four benchmarks, selecting the better response on only $40.0-49.1\%$ of disagreement prompts and reducing score relative to the better static model on all four. We trace the collapse to a structural decomposition: within the KLD-defined silent zone, KLD primarily measures the volume of disagreement with the reference (composite $\rho=+0.94$, $p<0.001$ on Qwen; $\rho=+0.55$, $p=0.03$ on Devstral), while its relationship to the direction of those disagreements is weak and task-conditional."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="nikolic-etal-2026-displacement">
<titleInfo>
<title>Displacement Is Not Direction: Evaluating Fidelity Metrics for Quantized LLM Deployment</title>
</titleInfo>
<name type="personal">
<namePart type="given">Milos</namePart>
<namePart type="family">Nikolic</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ali</namePart>
<namePart type="family">Hadi Zadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Enrique</namePart>
<namePart type="family">Torres Sanchez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andreas</namePart>
<namePart type="family">Moshovos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>Fidelity metrics such as per-token KL divergence (KLD) against a high-precision reference are often used as low-cost proxies for benchmark quality. We test this practice on 28 Qwen3.6-35B-A3B and 41 Devstral-Small-2-24B quantizations across downstream benchmarks. KLD strongly correlates with benchmark score over the full cohorts (ρ=-0.72 on Qwen and ρ=-0.86 on Devstral, both with p<0.001). However, among quantizations retaining at least 99% of measured BF16 composite quality, KLD provides no dependable ranking signal (ρ=+0.11, p=0.71, n=14 on Qwen; ρ=+0.02, p=0.93, n=16 on Devstral). This collapse persists across measurement variants spanning KLD aggregation, perplexity formulations, top-1 agreement, measurement corpora, and context lengths. At the per-prompt level, KLD provides only weak failure-prediction power and fails as a cross-model router across four benchmarks, selecting the better response on only 40.0-49.1% of disagreement prompts and reducing score relative to the better static model on all four. We trace the collapse to a structural decomposition: within the KLD-defined silent zone, KLD primarily measures the volume of disagreement with the reference (composite ρ=+0.94, p<0.001 on Qwen; ρ=+0.55, p=0.03 on Devstral), while its relationship to the direction of those disagreements is weak and task-conditional.</abstract>
<identifier type="citekey">nikolic-etal-2026-displacement</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.178/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>3025</start>
<end>3039</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Displacement Is Not Direction: Evaluating Fidelity Metrics for Quantized LLM Deployment
%A Nikolic, Milos
%A Hadi Zadeh, Ali
%A Torres Sanchez, Enrique
%A Moshovos, Andreas
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F nikolic-etal-2026-displacement
%X Fidelity metrics such as per-token KL divergence (KLD) against a high-precision reference are often used as low-cost proxies for benchmark quality. We test this practice on 28 Qwen3.6-35B-A3B and 41 Devstral-Small-2-24B quantizations across downstream benchmarks. KLD strongly correlates with benchmark score over the full cohorts (ρ=-0.72 on Qwen and ρ=-0.86 on Devstral, both with p<0.001). However, among quantizations retaining at least 99% of measured BF16 composite quality, KLD provides no dependable ranking signal (ρ=+0.11, p=0.71, n=14 on Qwen; ρ=+0.02, p=0.93, n=16 on Devstral). This collapse persists across measurement variants spanning KLD aggregation, perplexity formulations, top-1 agreement, measurement corpora, and context lengths. At the per-prompt level, KLD provides only weak failure-prediction power and fails as a cross-model router across four benchmarks, selecting the better response on only 40.0-49.1% of disagreement prompts and reducing score relative to the better static model on all four. We trace the collapse to a structural decomposition: within the KLD-defined silent zone, KLD primarily measures the volume of disagreement with the reference (composite ρ=+0.94, p<0.001 on Qwen; ρ=+0.55, p=0.03 on Devstral), while its relationship to the direction of those disagreements is weak and task-conditional.
%U https://aclanthology.org/2026.emnlp-industry.178/
%P 3025-3039
Markdown (Informal)
[Displacement Is Not Direction: Evaluating Fidelity Metrics for Quantized LLM Deployment](https://aclanthology.org/2026.emnlp-industry.178/) (Nikolic et al., EMNLP 2026)
ACL