@inproceedings{qiu-2026-bioscireview,
title = "{B}io{S}ci{R}eview-Bench: Benchmarking Coding Agents on Silent Scientific Bug Detection in Bioinformatics Code",
author = "Qiu, Junhao",
editor = "Kamalloo, Ehsan and
Gontier, Nicolas and
Rozanov, Nikolai and
Hashemzadeh, Maryam and
Arora, Akhil",
booktitle = "Proceedings of the Second Workshop for {RE}search on Agent Language Models ({REALM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.realm-1.65/",
pages = "947--956",
ISBN = "979-8-89176-485-9",
abstract = "Existing coding-agent benchmarks test functional correctness{---}whether code runs and returns expected outputs. Silent scientific methodology bugs pose a different challenge: they execute without error yet produce plausible, scientifically wrong results. We introduce BioSciReview-Bench, a benchmark of 200 manually reviewed synthetic bioinformatics code snippets (100 matched correct{--}buggy pairs) spanning five domains and six bug categories. Each snippet is presented in isolation without construction metadata; agents must detect whether a bug exists, localize it to a specific displayed line, and classify it by type. We evaluate six configurations across three model families and two agent harnesses. All achieve detection F1 of 0.94{--}0.99, but performance drops with stricter requirements: joint localization within three lines reaches 0.92{--}0.98 F1, and full diagnosis requiring exact line plus correct category falls to 0.74{--}0.89 F1, a gap of approximately 10{--}23 percentage points below detection. Filtering and quality-control bugs have the lowest average Full Diagnosis@{\ensuremath{\pm}}3 success rate, and agents sometimes confuse them with normalization errors. Detection performance is high on this controlled benchmark, while delivering actionable diagnoses remains substantially harder."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="qiu-2026-bioscireview">
<titleInfo>
<title>BioSciReview-Bench: Benchmarking Coding Agents on Silent Scientific Bug Detection in Bioinformatics Code</title>
</titleInfo>
<name type="personal">
<namePart type="given">Junhao</namePart>
<namePart type="family">Qiu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ehsan</namePart>
<namePart type="family">Kamalloo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicolas</namePart>
<namePart type="family">Gontier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikolai</namePart>
<namePart type="family">Rozanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maryam</namePart>
<namePart type="family">Hashemzadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akhil</namePart>
<namePart type="family">Arora</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-485-9</identifier>
</relatedItem>
<abstract>Existing coding-agent benchmarks test functional correctness—whether code runs and returns expected outputs. Silent scientific methodology bugs pose a different challenge: they execute without error yet produce plausible, scientifically wrong results. We introduce BioSciReview-Bench, a benchmark of 200 manually reviewed synthetic bioinformatics code snippets (100 matched correct–buggy pairs) spanning five domains and six bug categories. Each snippet is presented in isolation without construction metadata; agents must detect whether a bug exists, localize it to a specific displayed line, and classify it by type. We evaluate six configurations across three model families and two agent harnesses. All achieve detection F1 of 0.94–0.99, but performance drops with stricter requirements: joint localization within three lines reaches 0.92–0.98 F1, and full diagnosis requiring exact line plus correct category falls to 0.74–0.89 F1, a gap of approximately 10–23 percentage points below detection. Filtering and quality-control bugs have the lowest average Full Diagnosis@\ensuremath\pm3 success rate, and agents sometimes confuse them with normalization errors. Detection performance is high on this controlled benchmark, while delivering actionable diagnoses remains substantially harder.</abstract>
<identifier type="citekey">qiu-2026-bioscireview</identifier>
<location>
<url>https://aclanthology.org/2026.realm-1.65/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>947</start>
<end>956</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T BioSciReview-Bench: Benchmarking Coding Agents on Silent Scientific Bug Detection in Bioinformatics Code
%A Qiu, Junhao
%Y Kamalloo, Ehsan
%Y Gontier, Nicolas
%Y Rozanov, Nikolai
%Y Hashemzadeh, Maryam
%Y Arora, Akhil
%S Proceedings of the Second Workshop for REsearch on Agent Language Models (REALM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-485-9
%F qiu-2026-bioscireview
%X Existing coding-agent benchmarks test functional correctness—whether code runs and returns expected outputs. Silent scientific methodology bugs pose a different challenge: they execute without error yet produce plausible, scientifically wrong results. We introduce BioSciReview-Bench, a benchmark of 200 manually reviewed synthetic bioinformatics code snippets (100 matched correct–buggy pairs) spanning five domains and six bug categories. Each snippet is presented in isolation without construction metadata; agents must detect whether a bug exists, localize it to a specific displayed line, and classify it by type. We evaluate six configurations across three model families and two agent harnesses. All achieve detection F1 of 0.94–0.99, but performance drops with stricter requirements: joint localization within three lines reaches 0.92–0.98 F1, and full diagnosis requiring exact line plus correct category falls to 0.74–0.89 F1, a gap of approximately 10–23 percentage points below detection. Filtering and quality-control bugs have the lowest average Full Diagnosis@\ensuremath\pm3 success rate, and agents sometimes confuse them with normalization errors. Detection performance is high on this controlled benchmark, while delivering actionable diagnoses remains substantially harder.
%U https://aclanthology.org/2026.realm-1.65/
%P 947-956
Markdown (Informal)
[BioSciReview-Bench: Benchmarking Coding Agents on Silent Scientific Bug Detection in Bioinformatics Code](https://aclanthology.org/2026.realm-1.65/) (Qiu, REALM 2026)
ACL