@inproceedings{mcmahan-etal-2024-hassle,
title = "A Hassle-free Algorithm for Strong Differential Privacy in Federated Learning Systems",
author = "McMahan, Hugh Brendan and
Xu, Zheng and
Zhang, Yanxiang",
editor = "Dernoncourt, Franck and
Preo{\c{t}}iuc-Pietro, Daniel and
Shimorina, Anastasia",
booktitle = "Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = nov,
year = "2024",
address = "Miami, Florida, US",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2024.emnlp-industry.64",
pages = "842--865",
abstract = "Differential privacy (DP) and federated learning (FL) are combined as advanced privacy-preserving methods when training on-device language models in production mobile keyboard applications. DP-Follow-the-Regularized-Leader (DP-FTRL) algorithms, leveraging correlated noise mechanisms such as tree aggregation or matrix factorization, are widely used in practice for their superior privacy-utility trade-off and compatibility with FL systems. This paper presents a novel variant of DP-FTRL by adapting the recent theoretical advancements of the Buffered Linear Toeplitz (BLT) mechanism to multi-participant scenarios. In the FL setting, our BLT mechanism demonstrates enhanced privacy-utility trade-off and improved memory efficiency than the widely used tree aggregation mechanism. Moreover, BLT achieves comparable privacy and utility to the state-of-the-art banded matrix factorization mechanism, while significantly simplifying usage requirements and reducing memory. The flexibility of the BLT mechanism allows seamless integration with existing DP FL implementations in production environments. We evaluate the BLT-DP-FTRL algorithm on the StackOverflow dataset, serving as a research simulation benchmark, and across four on-device language model tasks in a production FL system. Our empirical results highlight the potential of the BLT mechanism to elevate the practicality and effectiveness of DP in real-world scenarios.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mcmahan-etal-2024-hassle">
<titleInfo>
<title>A Hassle-free Algorithm for Strong Differential Privacy in Federated Learning Systems</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hugh</namePart>
<namePart type="given">Brendan</namePart>
<namePart type="family">McMahan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zheng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yanxiang</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2024-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Franck</namePart>
<namePart type="family">Dernoncourt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Preoţiuc-Pietro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anastasia</namePart>
<namePart type="family">Shimorina</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Miami, Florida, US</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Differential privacy (DP) and federated learning (FL) are combined as advanced privacy-preserving methods when training on-device language models in production mobile keyboard applications. DP-Follow-the-Regularized-Leader (DP-FTRL) algorithms, leveraging correlated noise mechanisms such as tree aggregation or matrix factorization, are widely used in practice for their superior privacy-utility trade-off and compatibility with FL systems. This paper presents a novel variant of DP-FTRL by adapting the recent theoretical advancements of the Buffered Linear Toeplitz (BLT) mechanism to multi-participant scenarios. In the FL setting, our BLT mechanism demonstrates enhanced privacy-utility trade-off and improved memory efficiency than the widely used tree aggregation mechanism. Moreover, BLT achieves comparable privacy and utility to the state-of-the-art banded matrix factorization mechanism, while significantly simplifying usage requirements and reducing memory. The flexibility of the BLT mechanism allows seamless integration with existing DP FL implementations in production environments. We evaluate the BLT-DP-FTRL algorithm on the StackOverflow dataset, serving as a research simulation benchmark, and across four on-device language model tasks in a production FL system. Our empirical results highlight the potential of the BLT mechanism to elevate the practicality and effectiveness of DP in real-world scenarios.</abstract>
<identifier type="citekey">mcmahan-etal-2024-hassle</identifier>
<location>
<url>https://aclanthology.org/2024.emnlp-industry.64</url>
</location>
<part>
<date>2024-11</date>
<extent unit="page">
<start>842</start>
<end>865</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T A Hassle-free Algorithm for Strong Differential Privacy in Federated Learning Systems
%A McMahan, Hugh Brendan
%A Xu, Zheng
%A Zhang, Yanxiang
%Y Dernoncourt, Franck
%Y Preoţiuc-Pietro, Daniel
%Y Shimorina, Anastasia
%S Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2024
%8 November
%I Association for Computational Linguistics
%C Miami, Florida, US
%F mcmahan-etal-2024-hassle
%X Differential privacy (DP) and federated learning (FL) are combined as advanced privacy-preserving methods when training on-device language models in production mobile keyboard applications. DP-Follow-the-Regularized-Leader (DP-FTRL) algorithms, leveraging correlated noise mechanisms such as tree aggregation or matrix factorization, are widely used in practice for their superior privacy-utility trade-off and compatibility with FL systems. This paper presents a novel variant of DP-FTRL by adapting the recent theoretical advancements of the Buffered Linear Toeplitz (BLT) mechanism to multi-participant scenarios. In the FL setting, our BLT mechanism demonstrates enhanced privacy-utility trade-off and improved memory efficiency than the widely used tree aggregation mechanism. Moreover, BLT achieves comparable privacy and utility to the state-of-the-art banded matrix factorization mechanism, while significantly simplifying usage requirements and reducing memory. The flexibility of the BLT mechanism allows seamless integration with existing DP FL implementations in production environments. We evaluate the BLT-DP-FTRL algorithm on the StackOverflow dataset, serving as a research simulation benchmark, and across four on-device language model tasks in a production FL system. Our empirical results highlight the potential of the BLT mechanism to elevate the practicality and effectiveness of DP in real-world scenarios.
%U https://aclanthology.org/2024.emnlp-industry.64
%P 842-865
Markdown (Informal)
[A Hassle-free Algorithm for Strong Differential Privacy in Federated Learning Systems](https://aclanthology.org/2024.emnlp-industry.64) (McMahan et al., EMNLP 2024)
ACL