@inproceedings{peng-etal-2026-beyond-gradient,
title = "Beyond Gradient Egalitarianism: Dual-Adaptive {C}-{AUM} Framework for Few-Shot Fine-Tuning of Large Language Models",
author = "Peng, Qiaojuan and
Luo, Xiong and
Yuan, Yuqi and
Zhao, Wenbing",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1534/",
doi = "10.18653/2026.findings-emnlp.1534",
pages = "32641--32653",
ISBN = "979-8-89176-462-0",
abstract = "While large language models (LLMs) demonstrate strong text classification capabilities, adapting them to specialized, few-shot industrial domains remains challenging. Vanilla supervised fine-tuning (SFT) applies uniform weighting to training samples and may overfit easy instances while insufficiently optimizing ambiguous boundary samples. Static regularization may further limit adaptation by applying the same constraint throughout training. To address these limitations, we propose cumulative area under the margin (C-AUM), a fine-tuning framework tailored to autoregressive LLM classification. C-AUM tracks the cumulative margin of the final decision token as an instance-level difficulty signal and uses it to coordinate AUM-aware instance weighting with difficulty-aware dynamic LoRA regularization. We also construct and release an expert-verified few-shot text classification benchmark for the steel industry, comprising 1,070 samples across seven fine-grained categories. Evaluations across six representative LLMs show that C-AUM achieves comparable or improved performance over vanilla SFT, with a maximum macro F1 gain of 1.99 percentage points. Our dataset and code are publicly available at https://github.com/pengdada98/C-AUM."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="peng-etal-2026-beyond-gradient">
<titleInfo>
<title>Beyond Gradient Egalitarianism: Dual-Adaptive C-AUM Framework for Few-Shot Fine-Tuning of Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Qiaojuan</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiong</namePart>
<namePart type="family">Luo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuqi</namePart>
<namePart type="family">Yuan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenbing</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>While large language models (LLMs) demonstrate strong text classification capabilities, adapting them to specialized, few-shot industrial domains remains challenging. Vanilla supervised fine-tuning (SFT) applies uniform weighting to training samples and may overfit easy instances while insufficiently optimizing ambiguous boundary samples. Static regularization may further limit adaptation by applying the same constraint throughout training. To address these limitations, we propose cumulative area under the margin (C-AUM), a fine-tuning framework tailored to autoregressive LLM classification. C-AUM tracks the cumulative margin of the final decision token as an instance-level difficulty signal and uses it to coordinate AUM-aware instance weighting with difficulty-aware dynamic LoRA regularization. We also construct and release an expert-verified few-shot text classification benchmark for the steel industry, comprising 1,070 samples across seven fine-grained categories. Evaluations across six representative LLMs show that C-AUM achieves comparable or improved performance over vanilla SFT, with a maximum macro F1 gain of 1.99 percentage points. Our dataset and code are publicly available at https://github.com/pengdada98/C-AUM.</abstract>
<identifier type="citekey">peng-etal-2026-beyond-gradient</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1534</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1534/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>32641</start>
<end>32653</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Gradient Egalitarianism: Dual-Adaptive C-AUM Framework for Few-Shot Fine-Tuning of Large Language Models
%A Peng, Qiaojuan
%A Luo, Xiong
%A Yuan, Yuqi
%A Zhao, Wenbing
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F peng-etal-2026-beyond-gradient
%X While large language models (LLMs) demonstrate strong text classification capabilities, adapting them to specialized, few-shot industrial domains remains challenging. Vanilla supervised fine-tuning (SFT) applies uniform weighting to training samples and may overfit easy instances while insufficiently optimizing ambiguous boundary samples. Static regularization may further limit adaptation by applying the same constraint throughout training. To address these limitations, we propose cumulative area under the margin (C-AUM), a fine-tuning framework tailored to autoregressive LLM classification. C-AUM tracks the cumulative margin of the final decision token as an instance-level difficulty signal and uses it to coordinate AUM-aware instance weighting with difficulty-aware dynamic LoRA regularization. We also construct and release an expert-verified few-shot text classification benchmark for the steel industry, comprising 1,070 samples across seven fine-grained categories. Evaluations across six representative LLMs show that C-AUM achieves comparable or improved performance over vanilla SFT, with a maximum macro F1 gain of 1.99 percentage points. Our dataset and code are publicly available at https://github.com/pengdada98/C-AUM.
%R 10.18653/2026.findings-emnlp.1534
%U https://aclanthology.org/2026.findings-emnlp.1534/
%U https://doi.org/10.18653/2026.findings-emnlp.1534
%P 32641-32653
Markdown (Informal)
[Beyond Gradient Egalitarianism: Dual-Adaptive C-AUM Framework for Few-Shot Fine-Tuning of Large Language Models](https://aclanthology.org/2026.findings-emnlp.1534/) (Peng et al., Findings 2026)
ACL