@inproceedings{kim-etal-2026-coin,
title = "{COIN}: A Live, Large-scale Benchmark for Semantics-to-Input Code Reasoning",
author = "Kim, Jiho and
Zhang, Cen and
Kim, Youngjoon and
Kim, Dongkwan and
Lee, Brian Junekyu and
Jeong, Dae R. and
Kim, Taesoo",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2130/",
pages = "48428--48453",
ISBN = "979-8-89176-459-0",
abstract = "Constructing inputs that drive execution to chosen code locations underpins many dynamic-analysis tasks in software engineering and security. It is a generalized constraint-solving challenge that requires agents to reason about the semantic mapping between input and code. We built COIN, the first benchmark to systematically evaluate this ability in modern code agents. By leveraging OSS-Fuzz{'}s long-running coverage data and maintainer-written harnesses, COIN constructs reachability challenges across over 1000 real-world projects in nine languages, and is contamination-resistant and self-refreshing as upstream code and OSS-Fuzz coverage evolve. To keep challenges non-trivial yet solvable, COIN samples two target families: lines OSS-Fuzz has covered but baselines miss, and lines never be reached but with static evidence of a reachable harness path. Across five independent runs on 137 targets from 15 projects in six languages, the strongest agent (Claude Code with Opus 4.6) reaches on average only 26.6{\%} of the C/C++ targets and 39.7{\%} of the Go/JVM/Python/Rust targets, never half of either set in any run, and the weakest (DeepSeek V3.2) stays below 9{\%}, indicating substantial room for improvement."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kim-etal-2026-coin">
<titleInfo>
<title>COIN: A Live, Large-scale Benchmark for Semantics-to-Input Code Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jiho</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cen</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Youngjoon</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dongkwan</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Brian</namePart>
<namePart type="given">Junekyu</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dae</namePart>
<namePart type="given">R</namePart>
<namePart type="family">Jeong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Taesoo</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Constructing inputs that drive execution to chosen code locations underpins many dynamic-analysis tasks in software engineering and security. It is a generalized constraint-solving challenge that requires agents to reason about the semantic mapping between input and code. We built COIN, the first benchmark to systematically evaluate this ability in modern code agents. By leveraging OSS-Fuzz’s long-running coverage data and maintainer-written harnesses, COIN constructs reachability challenges across over 1000 real-world projects in nine languages, and is contamination-resistant and self-refreshing as upstream code and OSS-Fuzz coverage evolve. To keep challenges non-trivial yet solvable, COIN samples two target families: lines OSS-Fuzz has covered but baselines miss, and lines never be reached but with static evidence of a reachable harness path. Across five independent runs on 137 targets from 15 projects in six languages, the strongest agent (Claude Code with Opus 4.6) reaches on average only 26.6% of the C/C++ targets and 39.7% of the Go/JVM/Python/Rust targets, never half of either set in any run, and the weakest (DeepSeek V3.2) stays below 9%, indicating substantial room for improvement.</abstract>
<identifier type="citekey">kim-etal-2026-coin</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2130/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>48428</start>
<end>48453</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T COIN: A Live, Large-scale Benchmark for Semantics-to-Input Code Reasoning
%A Kim, Jiho
%A Zhang, Cen
%A Kim, Youngjoon
%A Kim, Dongkwan
%A Lee, Brian Junekyu
%A Jeong, Dae R.
%A Kim, Taesoo
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F kim-etal-2026-coin
%X Constructing inputs that drive execution to chosen code locations underpins many dynamic-analysis tasks in software engineering and security. It is a generalized constraint-solving challenge that requires agents to reason about the semantic mapping between input and code. We built COIN, the first benchmark to systematically evaluate this ability in modern code agents. By leveraging OSS-Fuzz’s long-running coverage data and maintainer-written harnesses, COIN constructs reachability challenges across over 1000 real-world projects in nine languages, and is contamination-resistant and self-refreshing as upstream code and OSS-Fuzz coverage evolve. To keep challenges non-trivial yet solvable, COIN samples two target families: lines OSS-Fuzz has covered but baselines miss, and lines never be reached but with static evidence of a reachable harness path. Across five independent runs on 137 targets from 15 projects in six languages, the strongest agent (Claude Code with Opus 4.6) reaches on average only 26.6% of the C/C++ targets and 39.7% of the Go/JVM/Python/Rust targets, never half of either set in any run, and the weakest (DeepSeek V3.2) stays below 9%, indicating substantial room for improvement.
%U https://aclanthology.org/2026.emnlp-main.2130/
%P 48428-48453
Markdown (Informal)
[COIN: A Live, Large-scale Benchmark for Semantics-to-Input Code Reasoning](https://aclanthology.org/2026.emnlp-main.2130/) (Kim et al., EMNLP 2026)
ACL
- Jiho Kim, Cen Zhang, Youngjoon Kim, Dongkwan Kim, Brian Junekyu Lee, Dae R. Jeong, and Taesoo Kim. 2026. COIN: A Live, Large-scale Benchmark for Semantics-to-Input Code Reasoning. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 48428–48453, Budapest, Hungary. Association for Computational Linguistics.