@inproceedings{yu-etal-2026-fhe,
title = "{FHE}-{C}ode{E}val: Benchmarking Large Language Models for Fully Homomorphic Encryption Code Generation",
author = "Yu, Qiuchu and
Li, Guangli and
Zhang, Shuoming and
Xu, Ruiyuan and
Cai, Qihan and
Xue, Jingling and
Feng, Xiaobing and
Cui, Huimin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.600/",
pages = "12657--12673",
ISBN = "979-8-89176-462-0",
abstract = "Large language models (LLMs) are increasingly used for code generation, but their ability to program specialized cryptographic systems remains poorly understood. Cheon-Kim-Kim-Song (CKKS), a fully homomorphic encryption (FHE) scheme supporting approximate arithmetic over packed real- and complex-valued vectors, is particularly relevant to machine-learning workloads. Prior work has not systematically examined the CKKS-specific challenges of LLM-generated encrypted programs. To address this gap, we introduce FHE-CodeEval, a 46-case benchmark that evaluates three core challenges: packing, nonlinear approximation, and level/scale management, as well as how models coordinate decisions across them. We further introduce communication-aware efficiency, an FHE-specific metric comparing generated programs with compiler baselines using server-side runtime and ciphertext footprint.\par We evaluate eight recent proprietary and open-weight LLMs on FHE-CodeEval. Across all 46 cases, the best observed scores reach 51.7{\%} Pass@1 and 69.6{\%} Pass@5 with iterative execution feedback, but only 7.8{\%} Pass@1 and 13.0{\%} Pass@5 in the one-shot setting. For efficiency, even the best-performing model achieves a geometric-mean communication-aware efficiency of only 0.44 over its valid rollouts in the compiler-comparable subset. These results reveal a substantial gap between functional correctness and efficiency in LLM-generated CKKS programs. Code is available at https://github.com/qiuchuy/FHE-CodeEval."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yu-etal-2026-fhe">
<titleInfo>
<title>FHE-CodeEval: Benchmarking Large Language Models for Fully Homomorphic Encryption Code Generation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Qiuchu</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Guangli</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shuoming</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruiyuan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qihan</namePart>
<namePart type="family">Cai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingling</namePart>
<namePart type="family">Xue</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaobing</namePart>
<namePart type="family">Feng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Huimin</namePart>
<namePart type="family">Cui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large language models (LLMs) are increasingly used for code generation, but their ability to program specialized cryptographic systems remains poorly understood. Cheon-Kim-Kim-Song (CKKS), a fully homomorphic encryption (FHE) scheme supporting approximate arithmetic over packed real- and complex-valued vectors, is particularly relevant to machine-learning workloads. Prior work has not systematically examined the CKKS-specific challenges of LLM-generated encrypted programs. To address this gap, we introduce FHE-CodeEval, a 46-case benchmark that evaluates three core challenges: packing, nonlinear approximation, and level/scale management, as well as how models coordinate decisions across them. We further introduce communication-aware efficiency, an FHE-specific metric comparing generated programs with compiler baselines using server-side runtime and ciphertext footprint.\par We evaluate eight recent proprietary and open-weight LLMs on FHE-CodeEval. Across all 46 cases, the best observed scores reach 51.7% Pass@1 and 69.6% Pass@5 with iterative execution feedback, but only 7.8% Pass@1 and 13.0% Pass@5 in the one-shot setting. For efficiency, even the best-performing model achieves a geometric-mean communication-aware efficiency of only 0.44 over its valid rollouts in the compiler-comparable subset. These results reveal a substantial gap between functional correctness and efficiency in LLM-generated CKKS programs. Code is available at https://github.com/qiuchuy/FHE-CodeEval.</abstract>
<identifier type="citekey">yu-etal-2026-fhe</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.600/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>12657</start>
<end>12673</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T FHE-CodeEval: Benchmarking Large Language Models for Fully Homomorphic Encryption Code Generation
%A Yu, Qiuchu
%A Li, Guangli
%A Zhang, Shuoming
%A Xu, Ruiyuan
%A Cai, Qihan
%A Xue, Jingling
%A Feng, Xiaobing
%A Cui, Huimin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F yu-etal-2026-fhe
%X Large language models (LLMs) are increasingly used for code generation, but their ability to program specialized cryptographic systems remains poorly understood. Cheon-Kim-Kim-Song (CKKS), a fully homomorphic encryption (FHE) scheme supporting approximate arithmetic over packed real- and complex-valued vectors, is particularly relevant to machine-learning workloads. Prior work has not systematically examined the CKKS-specific challenges of LLM-generated encrypted programs. To address this gap, we introduce FHE-CodeEval, a 46-case benchmark that evaluates three core challenges: packing, nonlinear approximation, and level/scale management, as well as how models coordinate decisions across them. We further introduce communication-aware efficiency, an FHE-specific metric comparing generated programs with compiler baselines using server-side runtime and ciphertext footprint.\par We evaluate eight recent proprietary and open-weight LLMs on FHE-CodeEval. Across all 46 cases, the best observed scores reach 51.7% Pass@1 and 69.6% Pass@5 with iterative execution feedback, but only 7.8% Pass@1 and 13.0% Pass@5 in the one-shot setting. For efficiency, even the best-performing model achieves a geometric-mean communication-aware efficiency of only 0.44 over its valid rollouts in the compiler-comparable subset. These results reveal a substantial gap between functional correctness and efficiency in LLM-generated CKKS programs. Code is available at https://github.com/qiuchuy/FHE-CodeEval.
%U https://aclanthology.org/2026.findings-emnlp.600/
%P 12657-12673
Markdown (Informal)
[FHE-CodeEval: Benchmarking Large Language Models for Fully Homomorphic Encryption Code Generation](https://aclanthology.org/2026.findings-emnlp.600/) (Yu et al., Findings 2026)
ACL
- Qiuchu Yu, Guangli Li, Shuoming Zhang, Ruiyuan Xu, Qihan Cai, Jingling Xue, Xiaobing Feng, and Huimin Cui. 2026. FHE-CodeEval: Benchmarking Large Language Models for Fully Homomorphic Encryption Code Generation. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 12657–12673, Budapest, Hungary. Association for Computational Linguistics.