@inproceedings{oamen-etal-2026-blockbench,
title = "{B}lock{B}ench: Contamination-Controlled Evaluation of Large Language Models for Code Analysis",
author = "Oamen, Paul Osemudiame and
Mene, Courage Ochuko and
Daniel, Abraham Ugbedeojo and
Mukherjee, Ananya and
Onobhayedo, Pius and
Zhao, Wei",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.154/",
pages = "3043--3062",
ISBN = "979-8-89176-444-6",
abstract = "How much of large language model (LLM) performance in code analysis reflects memorization of training data rather than genuine reasoning? We introduce BLOCKBENCH, a contamination-controlled evaluation framework that separates memorization effects from analytical capability in smart contract vulnerability detection. We evaluate seven frontier models on 316 vulnerable contracts augmented with 322 semantic-preserving transformations and 100 clean contracts. We observe a substantial performance gap across data exposure levels: detection accuracy drops from 86.5{\%} on likely-contaminated samples to 27.6{\%} on post-cutoff contracts, and further to 0.8{\%} on the most recently disclosed subset. A temporal analysis shows a strong exposure effect, where detection rates increase from 5.9{\%} to 84.7{\%} as contracts become more publicly visible, indicating that benchmark performance is heavily inflated by data contamination.\par We further identify a pattern matching paradox, where models can often locate security-relevant code regions but fail to explain why they are exploitable, revealing a form of explanation unfaithfulness in code understanding. In contrast, traditional static analyzers show complementary limitations: Slither compiles only 6.6{\%} of post-cutoff contracts, while Mythril compiles all inputs but fails to detect vulnerabilities. BlockBench generalizes to code understanding tasks where semantic equivalence can be verified, enabling more reliable evaluation under potential contamination."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="oamen-etal-2026-blockbench">
<titleInfo>
<title>BlockBench: Contamination-Controlled Evaluation of Large Language Models for Code Analysis</title>
</titleInfo>
<name type="personal">
<namePart type="given">Paul</namePart>
<namePart type="given">Osemudiame</namePart>
<namePart type="family">Oamen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Courage</namePart>
<namePart type="given">Ochuko</namePart>
<namePart type="family">Mene</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abraham</namePart>
<namePart type="given">Ugbedeojo</namePart>
<namePart type="family">Daniel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ananya</namePart>
<namePart type="family">Mukherjee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pius</namePart>
<namePart type="family">Onobhayedo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>How much of large language model (LLM) performance in code analysis reflects memorization of training data rather than genuine reasoning? We introduce BLOCKBENCH, a contamination-controlled evaluation framework that separates memorization effects from analytical capability in smart contract vulnerability detection. We evaluate seven frontier models on 316 vulnerable contracts augmented with 322 semantic-preserving transformations and 100 clean contracts. We observe a substantial performance gap across data exposure levels: detection accuracy drops from 86.5% on likely-contaminated samples to 27.6% on post-cutoff contracts, and further to 0.8% on the most recently disclosed subset. A temporal analysis shows a strong exposure effect, where detection rates increase from 5.9% to 84.7% as contracts become more publicly visible, indicating that benchmark performance is heavily inflated by data contamination.\par We further identify a pattern matching paradox, where models can often locate security-relevant code regions but fail to explain why they are exploitable, revealing a form of explanation unfaithfulness in code understanding. In contrast, traditional static analyzers show complementary limitations: Slither compiles only 6.6% of post-cutoff contracts, while Mythril compiles all inputs but fails to detect vulnerabilities. BlockBench generalizes to code understanding tasks where semantic equivalence can be verified, enabling more reliable evaluation under potential contamination.</abstract>
<identifier type="citekey">oamen-etal-2026-blockbench</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.154/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>3043</start>
<end>3062</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T BlockBench: Contamination-Controlled Evaluation of Large Language Models for Code Analysis
%A Oamen, Paul Osemudiame
%A Mene, Courage Ochuko
%A Daniel, Abraham Ugbedeojo
%A Mukherjee, Ananya
%A Onobhayedo, Pius
%A Zhao, Wei
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F oamen-etal-2026-blockbench
%X How much of large language model (LLM) performance in code analysis reflects memorization of training data rather than genuine reasoning? We introduce BLOCKBENCH, a contamination-controlled evaluation framework that separates memorization effects from analytical capability in smart contract vulnerability detection. We evaluate seven frontier models on 316 vulnerable contracts augmented with 322 semantic-preserving transformations and 100 clean contracts. We observe a substantial performance gap across data exposure levels: detection accuracy drops from 86.5% on likely-contaminated samples to 27.6% on post-cutoff contracts, and further to 0.8% on the most recently disclosed subset. A temporal analysis shows a strong exposure effect, where detection rates increase from 5.9% to 84.7% as contracts become more publicly visible, indicating that benchmark performance is heavily inflated by data contamination.\par We further identify a pattern matching paradox, where models can often locate security-relevant code regions but fail to explain why they are exploitable, revealing a form of explanation unfaithfulness in code understanding. In contrast, traditional static analyzers show complementary limitations: Slither compiles only 6.6% of post-cutoff contracts, while Mythril compiles all inputs but fails to detect vulnerabilities. BlockBench generalizes to code understanding tasks where semantic equivalence can be verified, enabling more reliable evaluation under potential contamination.
%U https://aclanthology.org/2026.findings-aacl.154/
%P 3043-3062
Markdown (Informal)
[BlockBench: Contamination-Controlled Evaluation of Large Language Models for Code Analysis](https://aclanthology.org/2026.findings-aacl.154/) (Oamen et al., Findings 2026)
ACL