@inproceedings{singh-2026-gets,
title = "Who Gets Annotated? Impact-Aware Active Learning for Robust Humanitarian Text Classification",
author = "Singh, Divyajot",
editor = "Atwell, Katherine and
Borah, Angana and
Dementieva, Daryna and
Kreiss, Elisa and
Kotonya, Neema and
Liu, Jiarui and
Olson, Liz and
Wan, Ruyuan and
Zhao, Jieyu",
booktitle = "Proceedings of the Fifth Workshop on {NLP} for Positive Impact ({NLP}4{PI})",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.nlp4pi-1.44/",
pages = "605--614",
ISBN = "979-8-89176-495-8",
abstract = "Humanitarian organizations must adapt text classifiers to new crises with small annotation budgets, yet conventional active learning optimizes model uncertainty without representing operational criticality or coverage across disaster events. We study an impact-aware batch acquisition policy that combines expected criticality, uncertainty, representation-space diversity, and event balance. Our evaluation crosses three recent open embedding backbones, three representation treatments, five acquisition strategies, five paired seeds, and total budgets of 50{--}500 labels. We separate exploratory development, event-held-out robustness on HumAID, and an untouched cross-dataset confirmation on CrisisMMD. On CrisisMMD, impact-aware selection with a conservative residual adapter improves Qwen{'}s area under the learning curve over random sampling by 0.089 for critical-category recall, 0.026 for macro-F1, and 0.032 for worst-event macro-F1. It raises the critical share of the 500-example labeled set from 3.4{\%} to 26.8{\%} and allocates annotations more evenly across events. Critical-recall gains also hold for Harrier and EmbeddingGemma on CrisisMMD and transfer to unseen HumAID events for Harrier, but not Qwen, exposing a consequential representation{--}dataset interaction. No configuration dominates every operational objective. Finally, 512-dimensional Qwen halves embedding storage while remaining within 0.009 AULC of its full representation across primary outcomes and both test settings. These results support impact-aware acquisition as a useful, but not always-on, policy and motivate reporting Pareto trade-offs rather than a single aggregate score."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="singh-2026-gets">
<titleInfo>
<title>Who Gets Annotated? Impact-Aware Active Learning for Robust Humanitarian Text Classification</title>
</titleInfo>
<name type="personal">
<namePart type="given">Divyajot</namePart>
<namePart type="family">Singh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Fifth Workshop on NLP for Positive Impact (NLP4PI)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Katherine</namePart>
<namePart type="family">Atwell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Angana</namePart>
<namePart type="family">Borah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daryna</namePart>
<namePart type="family">Dementieva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elisa</namePart>
<namePart type="family">Kreiss</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Neema</namePart>
<namePart type="family">Kotonya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiarui</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Liz</namePart>
<namePart type="family">Olson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruyuan</namePart>
<namePart type="family">Wan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jieyu</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-495-8</identifier>
</relatedItem>
<abstract>Humanitarian organizations must adapt text classifiers to new crises with small annotation budgets, yet conventional active learning optimizes model uncertainty without representing operational criticality or coverage across disaster events. We study an impact-aware batch acquisition policy that combines expected criticality, uncertainty, representation-space diversity, and event balance. Our evaluation crosses three recent open embedding backbones, three representation treatments, five acquisition strategies, five paired seeds, and total budgets of 50–500 labels. We separate exploratory development, event-held-out robustness on HumAID, and an untouched cross-dataset confirmation on CrisisMMD. On CrisisMMD, impact-aware selection with a conservative residual adapter improves Qwen’s area under the learning curve over random sampling by 0.089 for critical-category recall, 0.026 for macro-F1, and 0.032 for worst-event macro-F1. It raises the critical share of the 500-example labeled set from 3.4% to 26.8% and allocates annotations more evenly across events. Critical-recall gains also hold for Harrier and EmbeddingGemma on CrisisMMD and transfer to unseen HumAID events for Harrier, but not Qwen, exposing a consequential representation–dataset interaction. No configuration dominates every operational objective. Finally, 512-dimensional Qwen halves embedding storage while remaining within 0.009 AULC of its full representation across primary outcomes and both test settings. These results support impact-aware acquisition as a useful, but not always-on, policy and motivate reporting Pareto trade-offs rather than a single aggregate score.</abstract>
<identifier type="citekey">singh-2026-gets</identifier>
<location>
<url>https://aclanthology.org/2026.nlp4pi-1.44/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>605</start>
<end>614</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Who Gets Annotated? Impact-Aware Active Learning for Robust Humanitarian Text Classification
%A Singh, Divyajot
%Y Atwell, Katherine
%Y Borah, Angana
%Y Dementieva, Daryna
%Y Kreiss, Elisa
%Y Kotonya, Neema
%Y Liu, Jiarui
%Y Olson, Liz
%Y Wan, Ruyuan
%Y Zhao, Jieyu
%S Proceedings of the Fifth Workshop on NLP for Positive Impact (NLP4PI)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-495-8
%F singh-2026-gets
%X Humanitarian organizations must adapt text classifiers to new crises with small annotation budgets, yet conventional active learning optimizes model uncertainty without representing operational criticality or coverage across disaster events. We study an impact-aware batch acquisition policy that combines expected criticality, uncertainty, representation-space diversity, and event balance. Our evaluation crosses three recent open embedding backbones, three representation treatments, five acquisition strategies, five paired seeds, and total budgets of 50–500 labels. We separate exploratory development, event-held-out robustness on HumAID, and an untouched cross-dataset confirmation on CrisisMMD. On CrisisMMD, impact-aware selection with a conservative residual adapter improves Qwen’s area under the learning curve over random sampling by 0.089 for critical-category recall, 0.026 for macro-F1, and 0.032 for worst-event macro-F1. It raises the critical share of the 500-example labeled set from 3.4% to 26.8% and allocates annotations more evenly across events. Critical-recall gains also hold for Harrier and EmbeddingGemma on CrisisMMD and transfer to unseen HumAID events for Harrier, but not Qwen, exposing a consequential representation–dataset interaction. No configuration dominates every operational objective. Finally, 512-dimensional Qwen halves embedding storage while remaining within 0.009 AULC of its full representation across primary outcomes and both test settings. These results support impact-aware acquisition as a useful, but not always-on, policy and motivate reporting Pareto trade-offs rather than a single aggregate score.
%U https://aclanthology.org/2026.nlp4pi-1.44/
%P 605-614
Markdown (Informal)
[Who Gets Annotated? Impact-Aware Active Learning for Robust Humanitarian Text Classification](https://aclanthology.org/2026.nlp4pi-1.44/) (Singh, NLP4PI 2026)
ACL