@inproceedings{das-etal-2026-mince,
title = "{MINCE}: Shrinking {LLM} Evaluation Datasets via Few-Model {M}onte {C}arlo Calibration",
author = "Das, Devleena and
Patwari, Rajeev and
Bukka, Vikram Kumar and
Guggilla, Nithin Kumar and
Delaye, Elliott and
Sirasao, Ashish",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.84/",
pages = "1418--1428",
ISBN = "979-8-89176-460-6",
abstract = "Evaluating LLMs across many model variants{---}quantized, fine-tuned, or deployment-specific{---}requires running large benchmarks repeatedly, a process that can take tens of hours per model on edge hardware such as NPUs. Existing subset selection methods reduce this cost but depend on large calibration pools or learned prediction layers. We introduce MINCE (Monte Carlo Informed N-sizing for Compact Evaluation), which uses Monte Carlo simulation over per-item logs from a small set of calibration models to find the minimum subset size that bounds accuracy drift and then fixes a randomly sampled subset at that size, with no prediction layer needed. MINCE reduces IFEVAL by 54{\%}, MMLU by 89{\%}, GSM8K by 70{\%}, and MMLU-Pro by 88{\%} with maximum drift $\leq$2.62 pp on BF16 calibration models. The frozen subsets generalize to held-out GPU models with mean drift $\leq$1.40 pp and to INT4 NPU models with mean drift of 0.77{--}3.59 pp, while delivering evaluation speedups of up to 8.1$\times$ on the GPU models and evaluation speedups of 1.7{--}3.4$\times$ on the NPU models. The method is robust to calibration pool size and achieves lower drift than tinyBenchmarks (12$\times$ lower on MMLU, 3.3$\times$ on GSM8K) while using 42$\times$ fewer calibration models."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="das-etal-2026-mince">
<titleInfo>
<title>MINCE: Shrinking LLM Evaluation Datasets via Few-Model Monte Carlo Calibration</title>
</titleInfo>
<name type="personal">
<namePart type="given">Devleena</namePart>
<namePart type="family">Das</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rajeev</namePart>
<namePart type="family">Patwari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vikram</namePart>
<namePart type="given">Kumar</namePart>
<namePart type="family">Bukka</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nithin</namePart>
<namePart type="given">Kumar</namePart>
<namePart type="family">Guggilla</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elliott</namePart>
<namePart type="family">Delaye</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ashish</namePart>
<namePart type="family">Sirasao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>Evaluating LLMs across many model variants—quantized, fine-tuned, or deployment-specific—requires running large benchmarks repeatedly, a process that can take tens of hours per model on edge hardware such as NPUs. Existing subset selection methods reduce this cost but depend on large calibration pools or learned prediction layers. We introduce MINCE (Monte Carlo Informed N-sizing for Compact Evaluation), which uses Monte Carlo simulation over per-item logs from a small set of calibration models to find the minimum subset size that bounds accuracy drift and then fixes a randomly sampled subset at that size, with no prediction layer needed. MINCE reduces IFEVAL by 54%, MMLU by 89%, GSM8K by 70%, and MMLU-Pro by 88% with maximum drift łeq2.62 pp on BF16 calibration models. The frozen subsets generalize to held-out GPU models with mean drift łeq1.40 pp and to INT4 NPU models with mean drift of 0.77–3.59 pp, while delivering evaluation speedups of up to 8.1\times on the GPU models and evaluation speedups of 1.7–3.4\times on the NPU models. The method is robust to calibration pool size and achieves lower drift than tinyBenchmarks (12\times lower on MMLU, 3.3\times on GSM8K) while using 42\times fewer calibration models.</abstract>
<identifier type="citekey">das-etal-2026-mince</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.84/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1418</start>
<end>1428</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MINCE: Shrinking LLM Evaluation Datasets via Few-Model Monte Carlo Calibration
%A Das, Devleena
%A Patwari, Rajeev
%A Bukka, Vikram Kumar
%A Guggilla, Nithin Kumar
%A Delaye, Elliott
%A Sirasao, Ashish
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F das-etal-2026-mince
%X Evaluating LLMs across many model variants—quantized, fine-tuned, or deployment-specific—requires running large benchmarks repeatedly, a process that can take tens of hours per model on edge hardware such as NPUs. Existing subset selection methods reduce this cost but depend on large calibration pools or learned prediction layers. We introduce MINCE (Monte Carlo Informed N-sizing for Compact Evaluation), which uses Monte Carlo simulation over per-item logs from a small set of calibration models to find the minimum subset size that bounds accuracy drift and then fixes a randomly sampled subset at that size, with no prediction layer needed. MINCE reduces IFEVAL by 54%, MMLU by 89%, GSM8K by 70%, and MMLU-Pro by 88% with maximum drift łeq2.62 pp on BF16 calibration models. The frozen subsets generalize to held-out GPU models with mean drift łeq1.40 pp and to INT4 NPU models with mean drift of 0.77–3.59 pp, while delivering evaluation speedups of up to 8.1\times on the GPU models and evaluation speedups of 1.7–3.4\times on the NPU models. The method is robust to calibration pool size and achieves lower drift than tinyBenchmarks (12\times lower on MMLU, 3.3\times on GSM8K) while using 42\times fewer calibration models.
%U https://aclanthology.org/2026.emnlp-industry.84/
%P 1418-1428
Markdown (Informal)
[MINCE: Shrinking LLM Evaluation Datasets via Few-Model Monte Carlo Calibration](https://aclanthology.org/2026.emnlp-industry.84/) (Das et al., EMNLP 2026)
ACL