@inproceedings{dmonte-etal-2026-sunder,
title = "{SUNDER}: Selective Unmasking for Text Understanding",
author = "Dmonte, Alphaeus and
Ranasinghe, Tharindu and
Zampieri, Marcos",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.825/",
pages = "19171--19188",
ISBN = "979-8-89176-459-0",
abstract = "Decoder-only large language models (LLMs) employ autoregressive text processing, attending exclusively to preceding tokens when generating subsequent ones. Although this architectural design excels in text generation applications, it constrains model performance on certain text understanding tasks. In this work, we propose SUNDER, a novel approach that enables partial bidirectional attention computation. We validate the proposed approach on three token spans identification tasks, such as Named Entity Recognition (NER), Aspect-based Sentiment Analysis (ABSA), and Definition Extraction (DEF). Results show that SUNDER improves LLM performance by up to 5 percentage points compared to traditional instruction-tuning. We further validate the robustness of our approach on multiple model families, model sizes, and prompting techniques. Our evaluation extends to multilingual scenarios and single-hop question answering (QA), confirming SUNDER{'}s effectiveness on multiple settings. The code is made available to the community."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="dmonte-etal-2026-sunder">
<titleInfo>
<title>SUNDER: Selective Unmasking for Text Understanding</title>
</titleInfo>
<name type="personal">
<namePart type="given">Alphaeus</namePart>
<namePart type="family">Dmonte</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tharindu</namePart>
<namePart type="family">Ranasinghe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marcos</namePart>
<namePart type="family">Zampieri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Decoder-only large language models (LLMs) employ autoregressive text processing, attending exclusively to preceding tokens when generating subsequent ones. Although this architectural design excels in text generation applications, it constrains model performance on certain text understanding tasks. In this work, we propose SUNDER, a novel approach that enables partial bidirectional attention computation. We validate the proposed approach on three token spans identification tasks, such as Named Entity Recognition (NER), Aspect-based Sentiment Analysis (ABSA), and Definition Extraction (DEF). Results show that SUNDER improves LLM performance by up to 5 percentage points compared to traditional instruction-tuning. We further validate the robustness of our approach on multiple model families, model sizes, and prompting techniques. Our evaluation extends to multilingual scenarios and single-hop question answering (QA), confirming SUNDER’s effectiveness on multiple settings. The code is made available to the community.</abstract>
<identifier type="citekey">dmonte-etal-2026-sunder</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.825/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>19171</start>
<end>19188</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SUNDER: Selective Unmasking for Text Understanding
%A Dmonte, Alphaeus
%A Ranasinghe, Tharindu
%A Zampieri, Marcos
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F dmonte-etal-2026-sunder
%X Decoder-only large language models (LLMs) employ autoregressive text processing, attending exclusively to preceding tokens when generating subsequent ones. Although this architectural design excels in text generation applications, it constrains model performance on certain text understanding tasks. In this work, we propose SUNDER, a novel approach that enables partial bidirectional attention computation. We validate the proposed approach on three token spans identification tasks, such as Named Entity Recognition (NER), Aspect-based Sentiment Analysis (ABSA), and Definition Extraction (DEF). Results show that SUNDER improves LLM performance by up to 5 percentage points compared to traditional instruction-tuning. We further validate the robustness of our approach on multiple model families, model sizes, and prompting techniques. Our evaluation extends to multilingual scenarios and single-hop question answering (QA), confirming SUNDER’s effectiveness on multiple settings. The code is made available to the community.
%U https://aclanthology.org/2026.emnlp-main.825/
%P 19171-19188
Markdown (Informal)
[SUNDER: Selective Unmasking for Text Understanding](https://aclanthology.org/2026.emnlp-main.825/) (Dmonte et al., EMNLP 2026)
ACL
- Alphaeus Dmonte, Tharindu Ranasinghe, and Marcos Zampieri. 2026. SUNDER: Selective Unmasking for Text Understanding. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 19171–19188, Budapest, Hungary. Association for Computational Linguistics.