@inproceedings{guo-etal-2026-kpocbench,
title = "{KP}oc{B}ench: Benchmarking and Measuring Code Reasoning of Language Models in Linux Kernel",
author = "Guo, Ziyi and
Pei, Kexin and
Chen, Yan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1853/",
pages = "42131--42145",
ISBN = "979-8-89176-459-0",
abstract = "Large Language Model (LLM) agents increasingly act by generating, executing, and revising code, making code reasoning essential for understanding program state, diagnosing failures, and repairing behavior. Existing evaluations often measure end-to-end software engineering outcomes, obscuring which reasoning capabilities limit agents in complex, execution-grounded systems. We study this problem through Proof-of-Concept (PoC) generation, where an agent synthesizes an executable trigger that reproduces a reported crash or sanitizer failure. We instantiate this task in the Linux kernel, whose scale, statefulness, pointer-heavy value flow, memory layout, and concurrency stress long-horizon code reasoning. We decompose kernel PoC generation into four capabilities: state-machine reasoning, dynamic memory layout reasoning, value-flow reasoning, and concurrency reasoning. We build KPocInfra to emit branch-level execution telemetry and curate kPocBench, a benchmark of 181 reproducible Linux kernel KASAN cases from Syzbot. On kPocBench, state-machine and value-flow reasoning explain most failures, while capability-scaffolded evidence improves the best success rate from 50{\%} to 61{\%} and moves 29 unsolved attempts deeper along vulnerable execution paths. These results position kernel PoC generation as a practical stress test for LLM code reasoning in large-scale systems software."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="guo-etal-2026-kpocbench">
<titleInfo>
<title>KPocBench: Benchmarking and Measuring Code Reasoning of Language Models in Linux Kernel</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ziyi</namePart>
<namePart type="family">Guo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kexin</namePart>
<namePart type="family">Pei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yan</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large Language Model (LLM) agents increasingly act by generating, executing, and revising code, making code reasoning essential for understanding program state, diagnosing failures, and repairing behavior. Existing evaluations often measure end-to-end software engineering outcomes, obscuring which reasoning capabilities limit agents in complex, execution-grounded systems. We study this problem through Proof-of-Concept (PoC) generation, where an agent synthesizes an executable trigger that reproduces a reported crash or sanitizer failure. We instantiate this task in the Linux kernel, whose scale, statefulness, pointer-heavy value flow, memory layout, and concurrency stress long-horizon code reasoning. We decompose kernel PoC generation into four capabilities: state-machine reasoning, dynamic memory layout reasoning, value-flow reasoning, and concurrency reasoning. We build KPocInfra to emit branch-level execution telemetry and curate kPocBench, a benchmark of 181 reproducible Linux kernel KASAN cases from Syzbot. On kPocBench, state-machine and value-flow reasoning explain most failures, while capability-scaffolded evidence improves the best success rate from 50% to 61% and moves 29 unsolved attempts deeper along vulnerable execution paths. These results position kernel PoC generation as a practical stress test for LLM code reasoning in large-scale systems software.</abstract>
<identifier type="citekey">guo-etal-2026-kpocbench</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1853/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>42131</start>
<end>42145</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T KPocBench: Benchmarking and Measuring Code Reasoning of Language Models in Linux Kernel
%A Guo, Ziyi
%A Pei, Kexin
%A Chen, Yan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F guo-etal-2026-kpocbench
%X Large Language Model (LLM) agents increasingly act by generating, executing, and revising code, making code reasoning essential for understanding program state, diagnosing failures, and repairing behavior. Existing evaluations often measure end-to-end software engineering outcomes, obscuring which reasoning capabilities limit agents in complex, execution-grounded systems. We study this problem through Proof-of-Concept (PoC) generation, where an agent synthesizes an executable trigger that reproduces a reported crash or sanitizer failure. We instantiate this task in the Linux kernel, whose scale, statefulness, pointer-heavy value flow, memory layout, and concurrency stress long-horizon code reasoning. We decompose kernel PoC generation into four capabilities: state-machine reasoning, dynamic memory layout reasoning, value-flow reasoning, and concurrency reasoning. We build KPocInfra to emit branch-level execution telemetry and curate kPocBench, a benchmark of 181 reproducible Linux kernel KASAN cases from Syzbot. On kPocBench, state-machine and value-flow reasoning explain most failures, while capability-scaffolded evidence improves the best success rate from 50% to 61% and moves 29 unsolved attempts deeper along vulnerable execution paths. These results position kernel PoC generation as a practical stress test for LLM code reasoning in large-scale systems software.
%U https://aclanthology.org/2026.emnlp-main.1853/
%P 42131-42145
Markdown (Informal)
[KPocBench: Benchmarking and Measuring Code Reasoning of Language Models in Linux Kernel](https://aclanthology.org/2026.emnlp-main.1853/) (Guo et al., EMNLP 2026)
ACL