@inproceedings{yuan-etal-2026-gradfiltering,
title = "{GRADFILTERING}: Uncertainty-Aware Gradient Signal-to-Noise Data Selection for Instruction Tuning",
author = "Yuan, Zhihang and
Yue, Chengyu and
Huang, Long and
Ou, Litu and
Shi, Lei",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2297/",
pages = "52236--52247",
ISBN = "979-8-89176-459-0",
abstract = "Modern instruction-tuning mixtures are increasingly large, redundant, and heterogeneous, making full-data supervised fine-tuning costly and often unnecessary. We propose GRADFILTERING, an objective-agnostic, validation-free data selection method that scores examples using uncertainty-aware proxy training dynamics. GRADFILTERING trains a lightweight proxy model with a LoRA ensemble and ranks examples by a Gradient Signal-to-Noise Ratio (G-SNR), which combines early-to-late gradient drop with late-stage ensemble disagreement. Because selection requires neither target validation examples nor target-model gradients, the selected subsets can be reused across downstream targets. Across Alpaca and Tulu-3-52K, dense LLaMA-2-7B, LLaMA-3-8B, and Qwen3-8B targets, and a Qwen3-30B-A3B MoE target, GRADFILTERING-selected 5{--}15{\%} subsets consistently outperform random selection and are competitive with the target-aware LESS-T reference while remaining validation-free. We further evaluate broad model capabilities on objective benchmarks and analyze proxy choice, gradient windows, ensemble size, human preferences, and selection cost, demonstrating the practical utility of early proxy dynamics for reusable, compact instruction-tuning data selection. Code is available at this GitHub repository."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yuan-etal-2026-gradfiltering">
<titleInfo>
<title>GRADFILTERING: Uncertainty-Aware Gradient Signal-to-Noise Data Selection for Instruction Tuning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zhihang</namePart>
<namePart type="family">Yuan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chengyu</namePart>
<namePart type="family">Yue</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Long</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Litu</namePart>
<namePart type="family">Ou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lei</namePart>
<namePart type="family">Shi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Modern instruction-tuning mixtures are increasingly large, redundant, and heterogeneous, making full-data supervised fine-tuning costly and often unnecessary. We propose GRADFILTERING, an objective-agnostic, validation-free data selection method that scores examples using uncertainty-aware proxy training dynamics. GRADFILTERING trains a lightweight proxy model with a LoRA ensemble and ranks examples by a Gradient Signal-to-Noise Ratio (G-SNR), which combines early-to-late gradient drop with late-stage ensemble disagreement. Because selection requires neither target validation examples nor target-model gradients, the selected subsets can be reused across downstream targets. Across Alpaca and Tulu-3-52K, dense LLaMA-2-7B, LLaMA-3-8B, and Qwen3-8B targets, and a Qwen3-30B-A3B MoE target, GRADFILTERING-selected 5–15% subsets consistently outperform random selection and are competitive with the target-aware LESS-T reference while remaining validation-free. We further evaluate broad model capabilities on objective benchmarks and analyze proxy choice, gradient windows, ensemble size, human preferences, and selection cost, demonstrating the practical utility of early proxy dynamics for reusable, compact instruction-tuning data selection. Code is available at this GitHub repository.</abstract>
<identifier type="citekey">yuan-etal-2026-gradfiltering</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2297/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>52236</start>
<end>52247</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T GRADFILTERING: Uncertainty-Aware Gradient Signal-to-Noise Data Selection for Instruction Tuning
%A Yuan, Zhihang
%A Yue, Chengyu
%A Huang, Long
%A Ou, Litu
%A Shi, Lei
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F yuan-etal-2026-gradfiltering
%X Modern instruction-tuning mixtures are increasingly large, redundant, and heterogeneous, making full-data supervised fine-tuning costly and often unnecessary. We propose GRADFILTERING, an objective-agnostic, validation-free data selection method that scores examples using uncertainty-aware proxy training dynamics. GRADFILTERING trains a lightweight proxy model with a LoRA ensemble and ranks examples by a Gradient Signal-to-Noise Ratio (G-SNR), which combines early-to-late gradient drop with late-stage ensemble disagreement. Because selection requires neither target validation examples nor target-model gradients, the selected subsets can be reused across downstream targets. Across Alpaca and Tulu-3-52K, dense LLaMA-2-7B, LLaMA-3-8B, and Qwen3-8B targets, and a Qwen3-30B-A3B MoE target, GRADFILTERING-selected 5–15% subsets consistently outperform random selection and are competitive with the target-aware LESS-T reference while remaining validation-free. We further evaluate broad model capabilities on objective benchmarks and analyze proxy choice, gradient windows, ensemble size, human preferences, and selection cost, demonstrating the practical utility of early proxy dynamics for reusable, compact instruction-tuning data selection. Code is available at this GitHub repository.
%U https://aclanthology.org/2026.emnlp-main.2297/
%P 52236-52247
Markdown (Informal)
[GRADFILTERING: Uncertainty-Aware Gradient Signal-to-Noise Data Selection for Instruction Tuning](https://aclanthology.org/2026.emnlp-main.2297/) (Yuan et al., EMNLP 2026)
ACL