@inproceedings{dente-etal-2026-llms,
title = "Do {LLM}s Understand Backend Data-Layer Semantics? A Controlled Study of Cross-Endpoint Issues",
author = "Dente, Francesco and
Satriani, Dario and
Santoro, Donatello and
Veltri, Enzo and
Papotti, Paolo",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2154/",
pages = "49005--49019",
ISBN = "979-8-89176-459-0",
abstract = "Evaluating Large Language Models (LLMs) on code review requires moving beyond local syntax to assess system-wide semantic reasoning. We investigate whether LLMs can detect semantic data-layer issues: latent bugs requiring cross-endpoint reasoning over persistent state. We introduce a controlled benchmark of 216 minimal database-backed tasks, derived from 18 independent hand-written scenarios, to evaluate five frontier and open-weights models under two settings: open-ended review versus targeted detection. Our evaluation reveals a stark discovery gap: while models reliably recognize explicitly described issues, they struggle to discover them unprompted, with average F1 dropping from 0.91 to 0.85 for the easiest category and from 0.73 to 0.03 for the hardest. Furthermore, models exhibit a hierarchy of detectability across issue types, and for retention issues, a compliance bias where explicit definitions lead to over-flagging of structurally similar but semantically correct code. Our findings demonstrate that targeted defect detection substantially overestimates true open-ended review capabilities, underscoring the need to evaluate LLMs on latent intent recovery."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="dente-etal-2026-llms">
<titleInfo>
<title>Do LLMs Understand Backend Data-Layer Semantics? A Controlled Study of Cross-Endpoint Issues</title>
</titleInfo>
<name type="personal">
<namePart type="given">Francesco</namePart>
<namePart type="family">Dente</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dario</namePart>
<namePart type="family">Satriani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Donatello</namePart>
<namePart type="family">Santoro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Enzo</namePart>
<namePart type="family">Veltri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Paolo</namePart>
<namePart type="family">Papotti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Evaluating Large Language Models (LLMs) on code review requires moving beyond local syntax to assess system-wide semantic reasoning. We investigate whether LLMs can detect semantic data-layer issues: latent bugs requiring cross-endpoint reasoning over persistent state. We introduce a controlled benchmark of 216 minimal database-backed tasks, derived from 18 independent hand-written scenarios, to evaluate five frontier and open-weights models under two settings: open-ended review versus targeted detection. Our evaluation reveals a stark discovery gap: while models reliably recognize explicitly described issues, they struggle to discover them unprompted, with average F1 dropping from 0.91 to 0.85 for the easiest category and from 0.73 to 0.03 for the hardest. Furthermore, models exhibit a hierarchy of detectability across issue types, and for retention issues, a compliance bias where explicit definitions lead to over-flagging of structurally similar but semantically correct code. Our findings demonstrate that targeted defect detection substantially overestimates true open-ended review capabilities, underscoring the need to evaluate LLMs on latent intent recovery.</abstract>
<identifier type="citekey">dente-etal-2026-llms</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2154/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>49005</start>
<end>49019</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Do LLMs Understand Backend Data-Layer Semantics? A Controlled Study of Cross-Endpoint Issues
%A Dente, Francesco
%A Satriani, Dario
%A Santoro, Donatello
%A Veltri, Enzo
%A Papotti, Paolo
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F dente-etal-2026-llms
%X Evaluating Large Language Models (LLMs) on code review requires moving beyond local syntax to assess system-wide semantic reasoning. We investigate whether LLMs can detect semantic data-layer issues: latent bugs requiring cross-endpoint reasoning over persistent state. We introduce a controlled benchmark of 216 minimal database-backed tasks, derived from 18 independent hand-written scenarios, to evaluate five frontier and open-weights models under two settings: open-ended review versus targeted detection. Our evaluation reveals a stark discovery gap: while models reliably recognize explicitly described issues, they struggle to discover them unprompted, with average F1 dropping from 0.91 to 0.85 for the easiest category and from 0.73 to 0.03 for the hardest. Furthermore, models exhibit a hierarchy of detectability across issue types, and for retention issues, a compliance bias where explicit definitions lead to over-flagging of structurally similar but semantically correct code. Our findings demonstrate that targeted defect detection substantially overestimates true open-ended review capabilities, underscoring the need to evaluate LLMs on latent intent recovery.
%U https://aclanthology.org/2026.emnlp-main.2154/
%P 49005-49019
Markdown (Informal)
[Do LLMs Understand Backend Data-Layer Semantics? A Controlled Study of Cross-Endpoint Issues](https://aclanthology.org/2026.emnlp-main.2154/) (Dente et al., EMNLP 2026)
ACL