@inproceedings{malpani-2026-efficient,
title = "Efficient Adaptation of Small Language Models for Reliable Mathematical Reasoning",
author = "Malpani, Kanak Anil",
editor = "Wang, Yimu and
Choi, Yee Man and
Wu, Di and
Ouyang, Siqi and
Azadani, Mozhgan Nasr and
Fung, Yi R. (May)",
booktitle = "Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently ({G}round{LM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.groundlm-1.91/",
pages = "1221--1231",
ISBN = "979-8-89176-478-1",
abstract = "Efficient training and adaptation are central to deploying language models under resource constraints. We present a controlled comparison of six parameter-efficient fine-tuning (PEFT) methods{---}LoRA, QLoRA, DoRA, PiSSA, LoftQ, and AdaLoRA{---}across the full cross-product of three base models (Qwen2.5-1.5B, Qwen2.5-7B, Llama-3-8B) and two mathematical reasoning benchmarks (GSM8K, MATH), with every configuration trained on MetaMathQA under one identical budget and reported as mean {\ensuremath{\pm}} standard deviation over three seeds. Qwen2.5-1.5B + PiSSA reaches 71.40 {\ensuremath{\pm}} 0.27{\%} on GSM8K, exceeding the unadapted 7B base model (68.50 {\ensuremath{\pm}} 0.42{\%}; Welch{'}s t(3.4) = 10.06, p = 0.001). Significance testing shows that the method ordering{---}PiSSA {\ensuremath{>}} DoRA {\ensuremath{>}} LoRA {\ensuremath{\approx}} AdaLoRA {\ensuremath{\approx}} LoftQ {\ensuremath{\approx}} QLoRA{---}is identical across all three base models (Kendall{'}s {\ensuremath{\tau}} = 1.0), but that only the PiSSA and DoRA gains over LoRA survive multiple-comparison correction: the LoRA/QLoRA/LoftQ/AdaLoRA cluster is statistically indistinguishable at n=3. Four-bit QLoRA cuts peak training VRAM by up to 69.6{\%} (72.8{\%} with PagedAdamW), within one point of the 68.7{\%} predicted by a static-memory model, while PagedAdamW trades 15{--}20{\%} throughput for memory safety. We add rank and target-module ablations, a GSM8K error taxonomy, and the finding that PiSSA{'}s spectral initialization at r=16 outperforms LoRA at r=64, and we release the training, evaluation, and profiling code with the exact configuration for every run."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="malpani-2026-efficient">
<titleInfo>
<title>Efficient Adaptation of Small Language Models for Reliable Mathematical Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kanak</namePart>
<namePart type="given">Anil</namePart>
<namePart type="family">Malpani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yee</namePart>
<namePart type="given">Man</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siqi</namePart>
<namePart type="family">Ouyang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mozhgan</namePart>
<namePart type="given">Nasr</namePart>
<namePart type="family">Azadani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="given">R</namePart>
<namePart type="given">(May)</namePart>
<namePart type="family">Fung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-478-1</identifier>
</relatedItem>
<abstract>Efficient training and adaptation are central to deploying language models under resource constraints. We present a controlled comparison of six parameter-efficient fine-tuning (PEFT) methods—LoRA, QLoRA, DoRA, PiSSA, LoftQ, and AdaLoRA—across the full cross-product of three base models (Qwen2.5-1.5B, Qwen2.5-7B, Llama-3-8B) and two mathematical reasoning benchmarks (GSM8K, MATH), with every configuration trained on MetaMathQA under one identical budget and reported as mean \ensuremath\pm standard deviation over three seeds. Qwen2.5-1.5B + PiSSA reaches 71.40 \ensuremath\pm 0.27% on GSM8K, exceeding the unadapted 7B base model (68.50 \ensuremath\pm 0.42%; Welch’s t(3.4) = 10.06, p = 0.001). Significance testing shows that the method ordering—PiSSA \ensuremath> DoRA \ensuremath> LoRA \ensuremath\approx AdaLoRA \ensuremath\approx LoftQ \ensuremath\approx QLoRA—is identical across all three base models (Kendall’s \ensuremathτ = 1.0), but that only the PiSSA and DoRA gains over LoRA survive multiple-comparison correction: the LoRA/QLoRA/LoftQ/AdaLoRA cluster is statistically indistinguishable at n=3. Four-bit QLoRA cuts peak training VRAM by up to 69.6% (72.8% with PagedAdamW), within one point of the 68.7% predicted by a static-memory model, while PagedAdamW trades 15–20% throughput for memory safety. We add rank and target-module ablations, a GSM8K error taxonomy, and the finding that PiSSA’s spectral initialization at r=16 outperforms LoRA at r=64, and we release the training, evaluation, and profiling code with the exact configuration for every run.</abstract>
<identifier type="citekey">malpani-2026-efficient</identifier>
<location>
<url>https://aclanthology.org/2026.groundlm-1.91/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1221</start>
<end>1231</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Efficient Adaptation of Small Language Models for Reliable Mathematical Reasoning
%A Malpani, Kanak Anil
%Y Wang, Yimu
%Y Choi, Yee Man
%Y Wu, Di
%Y Ouyang, Siqi
%Y Azadani, Mozhgan Nasr
%Y Fung, Yi R. (May)
%S Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-478-1
%F malpani-2026-efficient
%X Efficient training and adaptation are central to deploying language models under resource constraints. We present a controlled comparison of six parameter-efficient fine-tuning (PEFT) methods—LoRA, QLoRA, DoRA, PiSSA, LoftQ, and AdaLoRA—across the full cross-product of three base models (Qwen2.5-1.5B, Qwen2.5-7B, Llama-3-8B) and two mathematical reasoning benchmarks (GSM8K, MATH), with every configuration trained on MetaMathQA under one identical budget and reported as mean \ensuremath\pm standard deviation over three seeds. Qwen2.5-1.5B + PiSSA reaches 71.40 \ensuremath\pm 0.27% on GSM8K, exceeding the unadapted 7B base model (68.50 \ensuremath\pm 0.42%; Welch’s t(3.4) = 10.06, p = 0.001). Significance testing shows that the method ordering—PiSSA \ensuremath> DoRA \ensuremath> LoRA \ensuremath\approx AdaLoRA \ensuremath\approx LoftQ \ensuremath\approx QLoRA—is identical across all three base models (Kendall’s \ensuremathτ = 1.0), but that only the PiSSA and DoRA gains over LoRA survive multiple-comparison correction: the LoRA/QLoRA/LoftQ/AdaLoRA cluster is statistically indistinguishable at n=3. Four-bit QLoRA cuts peak training VRAM by up to 69.6% (72.8% with PagedAdamW), within one point of the 68.7% predicted by a static-memory model, while PagedAdamW trades 15–20% throughput for memory safety. We add rank and target-module ablations, a GSM8K error taxonomy, and the finding that PiSSA’s spectral initialization at r=16 outperforms LoRA at r=64, and we release the training, evaluation, and profiling code with the exact configuration for every run.
%U https://aclanthology.org/2026.groundlm-1.91/
%P 1221-1231
Markdown (Informal)
[Efficient Adaptation of Small Language Models for Reliable Mathematical Reasoning](https://aclanthology.org/2026.groundlm-1.91/) (Malpani, GroundLM 2026)
ACL