@inproceedings{kapoor-khan-2026-execretrieval,
title = "{E}xec{R}etrieval: Measuring the Functional-Correctness Gap in Code-Embedding Retrieval",
author = "Kapoor, Aaryan and
Khan, Md Abdullah Al Hafiz",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2448/",
pages = "55517--55533",
ISBN = "979-8-89176-459-0",
abstract = "Embedding-based code retrieval is a core component of coding agents and retrieval-augmented code generation, where retrieving *correct* code matters more than retrieving lexically similar code. Existing code-retrieval benchmarks do not plant controlled, execution-verified single-edit variants of each query{'}s canonical implementation in the search pool, leaving the question of whether embeddings can functionally discriminate correct from near-clone-but-incorrect code unanswered in a retrieval setting. Resolving this requires a benchmark whose search pool itself contains the relevant counterfactuals {---} execution-verified buggy variants near-identical to each canonical {---} so that a retriever{'}s rank ordering can be directly tested for functional discrimination rather than topical or identity overlap. We introduce ExecRetrieval, 939 Python tasks each paired with one execution-verified canonical implementation and up to four execution-verified buggy distractors, each generated by a mechanical mutation making a single targeted edit, and evaluate 23 dense embedding configurations plus BM25 under provider-native invocation with paired McNemar tests and query-level bootstrap intervals. With near-clone counterfactuals in the pool, the top hosted system reaches exec@10 = 1.00 but only exec@1 = 0.331; rank-1 misses are paired buggy variants 91.5{--}99.4{\%} of the time across the four leading systems, and the canonical scores below at least one of its four paired distractors in 67{--}78{\%} of queries on the leading systems. The full dataset, execution oracle, embedding matrices, environment snapshot, and pairwise statistical tests are released at the URL in Appendix D."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kapoor-khan-2026-execretrieval">
<titleInfo>
<title>ExecRetrieval: Measuring the Functional-Correctness Gap in Code-Embedding Retrieval</title>
</titleInfo>
<name type="personal">
<namePart type="given">Aaryan</namePart>
<namePart type="family">Kapoor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Md</namePart>
<namePart type="given">Abdullah</namePart>
<namePart type="given">Al</namePart>
<namePart type="given">Hafiz</namePart>
<namePart type="family">Khan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Embedding-based code retrieval is a core component of coding agents and retrieval-augmented code generation, where retrieving *correct* code matters more than retrieving lexically similar code. Existing code-retrieval benchmarks do not plant controlled, execution-verified single-edit variants of each query’s canonical implementation in the search pool, leaving the question of whether embeddings can functionally discriminate correct from near-clone-but-incorrect code unanswered in a retrieval setting. Resolving this requires a benchmark whose search pool itself contains the relevant counterfactuals — execution-verified buggy variants near-identical to each canonical — so that a retriever’s rank ordering can be directly tested for functional discrimination rather than topical or identity overlap. We introduce ExecRetrieval, 939 Python tasks each paired with one execution-verified canonical implementation and up to four execution-verified buggy distractors, each generated by a mechanical mutation making a single targeted edit, and evaluate 23 dense embedding configurations plus BM25 under provider-native invocation with paired McNemar tests and query-level bootstrap intervals. With near-clone counterfactuals in the pool, the top hosted system reaches exec@10 = 1.00 but only exec@1 = 0.331; rank-1 misses are paired buggy variants 91.5–99.4% of the time across the four leading systems, and the canonical scores below at least one of its four paired distractors in 67–78% of queries on the leading systems. The full dataset, execution oracle, embedding matrices, environment snapshot, and pairwise statistical tests are released at the URL in Appendix D.</abstract>
<identifier type="citekey">kapoor-khan-2026-execretrieval</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2448/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>55517</start>
<end>55533</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T ExecRetrieval: Measuring the Functional-Correctness Gap in Code-Embedding Retrieval
%A Kapoor, Aaryan
%A Khan, Md Abdullah Al Hafiz
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F kapoor-khan-2026-execretrieval
%X Embedding-based code retrieval is a core component of coding agents and retrieval-augmented code generation, where retrieving *correct* code matters more than retrieving lexically similar code. Existing code-retrieval benchmarks do not plant controlled, execution-verified single-edit variants of each query’s canonical implementation in the search pool, leaving the question of whether embeddings can functionally discriminate correct from near-clone-but-incorrect code unanswered in a retrieval setting. Resolving this requires a benchmark whose search pool itself contains the relevant counterfactuals — execution-verified buggy variants near-identical to each canonical — so that a retriever’s rank ordering can be directly tested for functional discrimination rather than topical or identity overlap. We introduce ExecRetrieval, 939 Python tasks each paired with one execution-verified canonical implementation and up to four execution-verified buggy distractors, each generated by a mechanical mutation making a single targeted edit, and evaluate 23 dense embedding configurations plus BM25 under provider-native invocation with paired McNemar tests and query-level bootstrap intervals. With near-clone counterfactuals in the pool, the top hosted system reaches exec@10 = 1.00 but only exec@1 = 0.331; rank-1 misses are paired buggy variants 91.5–99.4% of the time across the four leading systems, and the canonical scores below at least one of its four paired distractors in 67–78% of queries on the leading systems. The full dataset, execution oracle, embedding matrices, environment snapshot, and pairwise statistical tests are released at the URL in Appendix D.
%U https://aclanthology.org/2026.emnlp-main.2448/
%P 55517-55533
Markdown (Informal)
[ExecRetrieval: Measuring the Functional-Correctness Gap in Code-Embedding Retrieval](https://aclanthology.org/2026.emnlp-main.2448/) (Kapoor & Khan, EMNLP 2026)
ACL