@inproceedings{al-abdi-ezzini-2026-code,
title = "Code Without Context: Can We Trust {LLM}s to Test Software from Informal Descriptions?",
author = "Al Abdi, Amneh and
Ezzini, Saad",
editor = "Mitkov, Ruslan and
Mu{\~n}oz, Rafael and
Lloret, Elena and
Ranasinghe, Tharindu and
Estevanell-Valladares, Ernesto L. and
Lamsiyah, Salima and
Montoyo, Andr{\'e}s and
Ezzini, Saad",
booktitle = "Proceedings of the Second International Conference on Natural Language Processing and Artificial Intelligence for Cyber Security",
month = jun,
year = "2026",
address = "Alicante, Spain",
publisher = "Department of Languages and Information Systems, University of Alicante",
url = "https://aclanthology.org/2026.nlpaics-1.5/",
pages = "46--54",
abstract = "Recent advances in Large Language Models (LLMs) have shown their strong abilities in several tasks, such as text understanding and generation. In several domains, such as software engineering, LLMs are increasingly used to automate related tasks. However, LLM-generated outputs still raise major concerns about robustness and reliability, especially in real-world use. These concerns are especially important in cases where the available inputs are informal natural language rather than structured inputs, which raises issues related to the trustworthiness and potential risks of LLM outputs. This paper aims to evaluate LLMs for the automated test case generation task in a weaker-input setting, relying solely on their understanding of these unstructured inputs. A dataset of 191 programming problems is used to evaluate a general-purpose LLM, GPT-5-mini, and a code-specialized LLM, Qwen2.5-Coder-7B. The generated test cases are executed on reference Python solutions and evaluated in terms of both test-level and problem-level pass rates. The results show that GPT-5-mini outperformed Qwen2.5-Coder-7B in terms of both performance metrics, with 63.72{\%} and 59.16{\%} for test-level and problem-level, respectively, while Qwen2.5-Coder-7B achieves only 21.62{\%} and 2.09{\%} test-level and problem-level pass rates, respectively. The results of this study indicate that both models have limitations in understanding the semantics of informal natural language descriptions, and provide early insight into the feasibility and limitations of LLM-based test case generation using only natural language descriptions. These results also raise concerns related to the safe deployment of LLMs and potential risks, including undetected incorrect interpretations, especially in automated workflows and safety-critical applications."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="al-abdi-ezzini-2026-code">
<titleInfo>
<title>Code Without Context: Can We Trust LLMs to Test Software from Informal Descriptions?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Amneh</namePart>
<namePart type="family">Al Abdi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Saad</namePart>
<namePart type="family">Ezzini</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-06</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second International Conference on Natural Language Processing and Artificial Intelligence for Cyber Security</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ruslan</namePart>
<namePart type="family">Mitkov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rafael</namePart>
<namePart type="family">Muñoz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elena</namePart>
<namePart type="family">Lloret</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tharindu</namePart>
<namePart type="family">Ranasinghe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ernesto</namePart>
<namePart type="given">L</namePart>
<namePart type="family">Estevanell-Valladares</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Salima</namePart>
<namePart type="family">Lamsiyah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andrés</namePart>
<namePart type="family">Montoyo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Saad</namePart>
<namePart type="family">Ezzini</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Department of Languages and Information Systems, University of Alicante</publisher>
<place>
<placeTerm type="text">Alicante, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Recent advances in Large Language Models (LLMs) have shown their strong abilities in several tasks, such as text understanding and generation. In several domains, such as software engineering, LLMs are increasingly used to automate related tasks. However, LLM-generated outputs still raise major concerns about robustness and reliability, especially in real-world use. These concerns are especially important in cases where the available inputs are informal natural language rather than structured inputs, which raises issues related to the trustworthiness and potential risks of LLM outputs. This paper aims to evaluate LLMs for the automated test case generation task in a weaker-input setting, relying solely on their understanding of these unstructured inputs. A dataset of 191 programming problems is used to evaluate a general-purpose LLM, GPT-5-mini, and a code-specialized LLM, Qwen2.5-Coder-7B. The generated test cases are executed on reference Python solutions and evaluated in terms of both test-level and problem-level pass rates. The results show that GPT-5-mini outperformed Qwen2.5-Coder-7B in terms of both performance metrics, with 63.72% and 59.16% for test-level and problem-level, respectively, while Qwen2.5-Coder-7B achieves only 21.62% and 2.09% test-level and problem-level pass rates, respectively. The results of this study indicate that both models have limitations in understanding the semantics of informal natural language descriptions, and provide early insight into the feasibility and limitations of LLM-based test case generation using only natural language descriptions. These results also raise concerns related to the safe deployment of LLMs and potential risks, including undetected incorrect interpretations, especially in automated workflows and safety-critical applications.</abstract>
<identifier type="citekey">al-abdi-ezzini-2026-code</identifier>
<location>
<url>https://aclanthology.org/2026.nlpaics-1.5/</url>
</location>
<part>
<date>2026-06</date>
<extent unit="page">
<start>46</start>
<end>54</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Code Without Context: Can We Trust LLMs to Test Software from Informal Descriptions?
%A Al Abdi, Amneh
%A Ezzini, Saad
%Y Mitkov, Ruslan
%Y Muñoz, Rafael
%Y Lloret, Elena
%Y Ranasinghe, Tharindu
%Y Estevanell-Valladares, Ernesto L.
%Y Lamsiyah, Salima
%Y Montoyo, Andrés
%Y Ezzini, Saad
%S Proceedings of the Second International Conference on Natural Language Processing and Artificial Intelligence for Cyber Security
%D 2026
%8 June
%I Department of Languages and Information Systems, University of Alicante
%C Alicante, Spain
%F al-abdi-ezzini-2026-code
%X Recent advances in Large Language Models (LLMs) have shown their strong abilities in several tasks, such as text understanding and generation. In several domains, such as software engineering, LLMs are increasingly used to automate related tasks. However, LLM-generated outputs still raise major concerns about robustness and reliability, especially in real-world use. These concerns are especially important in cases where the available inputs are informal natural language rather than structured inputs, which raises issues related to the trustworthiness and potential risks of LLM outputs. This paper aims to evaluate LLMs for the automated test case generation task in a weaker-input setting, relying solely on their understanding of these unstructured inputs. A dataset of 191 programming problems is used to evaluate a general-purpose LLM, GPT-5-mini, and a code-specialized LLM, Qwen2.5-Coder-7B. The generated test cases are executed on reference Python solutions and evaluated in terms of both test-level and problem-level pass rates. The results show that GPT-5-mini outperformed Qwen2.5-Coder-7B in terms of both performance metrics, with 63.72% and 59.16% for test-level and problem-level, respectively, while Qwen2.5-Coder-7B achieves only 21.62% and 2.09% test-level and problem-level pass rates, respectively. The results of this study indicate that both models have limitations in understanding the semantics of informal natural language descriptions, and provide early insight into the feasibility and limitations of LLM-based test case generation using only natural language descriptions. These results also raise concerns related to the safe deployment of LLMs and potential risks, including undetected incorrect interpretations, especially in automated workflows and safety-critical applications.
%U https://aclanthology.org/2026.nlpaics-1.5/
%P 46-54
Markdown (Informal)
[Code Without Context: Can We Trust LLMs to Test Software from Informal Descriptions?](https://aclanthology.org/2026.nlpaics-1.5/) (Al Abdi & Ezzini, NLPAICS 2026)
ACL