@inproceedings{sun-etal-2026-fg2,
title = "{FG}$^2$-{GDN}: Enhancing Long-Context Gated Delta Networks with Doubly Fine-Grained Control",
author = "Sun, Pingwei and
Hu, Yuxuan and
Tan, Jianchao and
Wang, Xue and
Zhang, Jiaqi and
Lu, Yifan and
Sun, Yerui and
Xie, Yuchen and
Cai, Xunliang",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2242/",
pages = "51013--51023",
ISBN = "979-8-89176-459-0",
abstract = "Linear attention mechanisms have emerged as promising alternatives to softmax attention, offering linear-time complexity during inference. Recent advances such as Gated DeltaNet (GDN) and Kimi Delta Attention (KDA) have demonstrated that the delta rule (an online gradient descent update) enables superior associative recall compared to simple additive updates. While KDA refined the coarse head-wise decay gate into channel-wise decay, the learning rate $ \beta_t $ in the delta update remains a scalar, limiting the model{'}s capacity for dimension-specific adaptation. We introduce FG$^2$-GDN, which replaces the scalar $ \beta_t $ with a channel-wise vector in a manner analogous to the transition from SGD to per-coordinate adaptive optimizers such as AdaGrad and Adam. We further propose FG$^2$-GDN+, which decouples the scaling for keys and values, enabling independent control of erasure strength and write strength. Experiments on synthetic and real-world benchmarks show that FG$^2$-GDN and its variant improve associative recall and long-context understanding over GDN and KDA, with comparable computational efficiency."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sun-etal-2026-fg2">
<titleInfo>
<title>FG²-GDN: Enhancing Long-Context Gated Delta Networks with Doubly Fine-Grained Control</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pingwei</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuxuan</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jianchao</namePart>
<namePart type="family">Tan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xue</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiaqi</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yifan</namePart>
<namePart type="family">Lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yerui</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuchen</namePart>
<namePart type="family">Xie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xunliang</namePart>
<namePart type="family">Cai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Linear attention mechanisms have emerged as promising alternatives to softmax attention, offering linear-time complexity during inference. Recent advances such as Gated DeltaNet (GDN) and Kimi Delta Attention (KDA) have demonstrated that the delta rule (an online gradient descent update) enables superior associative recall compared to simple additive updates. While KDA refined the coarse head-wise decay gate into channel-wise decay, the learning rate β_t in the delta update remains a scalar, limiting the model’s capacity for dimension-specific adaptation. We introduce FG²-GDN, which replaces the scalar β_t with a channel-wise vector in a manner analogous to the transition from SGD to per-coordinate adaptive optimizers such as AdaGrad and Adam. We further propose FG²-GDN+, which decouples the scaling for keys and values, enabling independent control of erasure strength and write strength. Experiments on synthetic and real-world benchmarks show that FG²-GDN and its variant improve associative recall and long-context understanding over GDN and KDA, with comparable computational efficiency.</abstract>
<identifier type="citekey">sun-etal-2026-fg2</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2242/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>51013</start>
<end>51023</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T FG²-GDN: Enhancing Long-Context Gated Delta Networks with Doubly Fine-Grained Control
%A Sun, Pingwei
%A Hu, Yuxuan
%A Tan, Jianchao
%A Wang, Xue
%A Zhang, Jiaqi
%A Lu, Yifan
%A Sun, Yerui
%A Xie, Yuchen
%A Cai, Xunliang
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F sun-etal-2026-fg2
%X Linear attention mechanisms have emerged as promising alternatives to softmax attention, offering linear-time complexity during inference. Recent advances such as Gated DeltaNet (GDN) and Kimi Delta Attention (KDA) have demonstrated that the delta rule (an online gradient descent update) enables superior associative recall compared to simple additive updates. While KDA refined the coarse head-wise decay gate into channel-wise decay, the learning rate β_t in the delta update remains a scalar, limiting the model’s capacity for dimension-specific adaptation. We introduce FG²-GDN, which replaces the scalar β_t with a channel-wise vector in a manner analogous to the transition from SGD to per-coordinate adaptive optimizers such as AdaGrad and Adam. We further propose FG²-GDN+, which decouples the scaling for keys and values, enabling independent control of erasure strength and write strength. Experiments on synthetic and real-world benchmarks show that FG²-GDN and its variant improve associative recall and long-context understanding over GDN and KDA, with comparable computational efficiency.
%U https://aclanthology.org/2026.emnlp-main.2242/
%P 51013-51023
Markdown (Informal)
[FG2-GDN: Enhancing Long-Context Gated Delta Networks with Doubly Fine-Grained Control](https://aclanthology.org/2026.emnlp-main.2242/) (Sun et al., EMNLP 2026)
ACL
- Pingwei Sun, Yuxuan Hu, Jianchao Tan, Xue Wang, Jiaqi Zhang, Yifan Lu, Yerui Sun, Yuchen Xie, and Xunliang Cai. 2026. FG2-GDN: Enhancing Long-Context Gated Delta Networks with Doubly Fine-Grained Control. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 51013–51023, Budapest, Hungary. Association for Computational Linguistics.