@inproceedings{kadasi-etal-2026-task,
title = "Task{--}Specificity Score: Measuring How Much Instructions Really Matter for Supervision",
author = "Kadasi, Pritam and
Upperwal, Abhishek and
Singh, Mayank",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1945/",
pages = "44255--44267",
ISBN = "979-8-89176-459-0",
abstract = "Instruction tuning is now the default way to train and adapt large language models, but many instruction{--}input{--}output pairs are only weakly specified: for a given input, the same output can remain plausible under several alternative instructions. This raises a simple question: does the instruction uniquely determine the target output? We propose the Task{--}Specificity Score (TSS) to quantify how much an instruction matters for predicting the target output by contrasting the true instruction against plausible alternatives for the same input. We further introduce TSS++, which uses hard alternatives and a small quality term to mitigate easy-negative effects. Across three instruction datasets (Alpaca, Dolly-15K, NI-20) and three open LLMs (Gemma, Llama, Qwen), our strongest evidence appears in the tightest budget regime, particularly at 5{\%} retention. In this setting, TSS and TSS++ consistently outperform existing selection baselines across most model{--}dataset combinations. These results show that task specificity is a useful signal for low-budget instruction-data selection and complements existing filters such as perplexity and IFD."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kadasi-etal-2026-task">
<titleInfo>
<title>Task–Specificity Score: Measuring How Much Instructions Really Matter for Supervision</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pritam</namePart>
<namePart type="family">Kadasi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abhishek</namePart>
<namePart type="family">Upperwal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mayank</namePart>
<namePart type="family">Singh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Instruction tuning is now the default way to train and adapt large language models, but many instruction–input–output pairs are only weakly specified: for a given input, the same output can remain plausible under several alternative instructions. This raises a simple question: does the instruction uniquely determine the target output? We propose the Task–Specificity Score (TSS) to quantify how much an instruction matters for predicting the target output by contrasting the true instruction against plausible alternatives for the same input. We further introduce TSS++, which uses hard alternatives and a small quality term to mitigate easy-negative effects. Across three instruction datasets (Alpaca, Dolly-15K, NI-20) and three open LLMs (Gemma, Llama, Qwen), our strongest evidence appears in the tightest budget regime, particularly at 5% retention. In this setting, TSS and TSS++ consistently outperform existing selection baselines across most model–dataset combinations. These results show that task specificity is a useful signal for low-budget instruction-data selection and complements existing filters such as perplexity and IFD.</abstract>
<identifier type="citekey">kadasi-etal-2026-task</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1945/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>44255</start>
<end>44267</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Task–Specificity Score: Measuring How Much Instructions Really Matter for Supervision
%A Kadasi, Pritam
%A Upperwal, Abhishek
%A Singh, Mayank
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F kadasi-etal-2026-task
%X Instruction tuning is now the default way to train and adapt large language models, but many instruction–input–output pairs are only weakly specified: for a given input, the same output can remain plausible under several alternative instructions. This raises a simple question: does the instruction uniquely determine the target output? We propose the Task–Specificity Score (TSS) to quantify how much an instruction matters for predicting the target output by contrasting the true instruction against plausible alternatives for the same input. We further introduce TSS++, which uses hard alternatives and a small quality term to mitigate easy-negative effects. Across three instruction datasets (Alpaca, Dolly-15K, NI-20) and three open LLMs (Gemma, Llama, Qwen), our strongest evidence appears in the tightest budget regime, particularly at 5% retention. In this setting, TSS and TSS++ consistently outperform existing selection baselines across most model–dataset combinations. These results show that task specificity is a useful signal for low-budget instruction-data selection and complements existing filters such as perplexity and IFD.
%U https://aclanthology.org/2026.emnlp-main.1945/
%P 44255-44267
Markdown (Informal)
[Task–Specificity Score: Measuring How Much Instructions Really Matter for Supervision](https://aclanthology.org/2026.emnlp-main.1945/) (Kadasi et al., EMNLP 2026)
ACL