@inproceedings{quang-etal-2026-xrepotest,
title = "{XR}epo{T}est: Benchmarking Multilingual Repository-Level Unit Test Generation for Large Language Models",
author = "Quang, Dung Le and
Van, Dong Cao and
Hai, Nam Le and
Van, Linh Ngo and
Bui, Thi-Mai-Anh and
Nguyen, Phuong T.",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.274/",
pages = "6224--6262",
ISBN = "979-8-89176-459-0",
abstract = "Large language models (LLMs) have shown promise for automated unit test generation, but existing evaluations largely rely on standalone settings and a narrow set of programming languages, overestimating real-world readiness. We introduce XRepoTest, a multilingual, repository-level benchmark for unit test generation spanning five underexplored languages: Rust, Go, Julia, PHP, and Ruby. XRepoTest evaluates tests under realistic repository constraints using a containerized execution framework and multiple context augmentation strategies, including file-level, LSP-based, and retrieval-based context. Beyond standard metrics such as test pass rate and coverage, we propose Invocation Rate (IR) to assess whether generated tests meaningfully exercise the intended functionality. Experiments with 14 state-of-the-art LLMs (e.g., Claude 4.5, GPT-5.2, DeepSeek V4-pro, and Qwen families) reveal a substantial gap between standalone and repository-level performance, as well as trade-offs between richer context and test reliability. Overall, XRepoTest provides a challenging and informative benchmark to advance scalable, robust, and multilingual unit test generation in realistic software environments."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="quang-etal-2026-xrepotest">
<titleInfo>
<title>XRepoTest: Benchmarking Multilingual Repository-Level Unit Test Generation for Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dung</namePart>
<namePart type="given">Le</namePart>
<namePart type="family">Quang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dong</namePart>
<namePart type="given">Cao</namePart>
<namePart type="family">Van</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nam</namePart>
<namePart type="given">Le</namePart>
<namePart type="family">Hai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Linh</namePart>
<namePart type="given">Ngo</namePart>
<namePart type="family">Van</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thi-Mai-Anh</namePart>
<namePart type="family">Bui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Phuong</namePart>
<namePart type="given">T</namePart>
<namePart type="family">Nguyen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large language models (LLMs) have shown promise for automated unit test generation, but existing evaluations largely rely on standalone settings and a narrow set of programming languages, overestimating real-world readiness. We introduce XRepoTest, a multilingual, repository-level benchmark for unit test generation spanning five underexplored languages: Rust, Go, Julia, PHP, and Ruby. XRepoTest evaluates tests under realistic repository constraints using a containerized execution framework and multiple context augmentation strategies, including file-level, LSP-based, and retrieval-based context. Beyond standard metrics such as test pass rate and coverage, we propose Invocation Rate (IR) to assess whether generated tests meaningfully exercise the intended functionality. Experiments with 14 state-of-the-art LLMs (e.g., Claude 4.5, GPT-5.2, DeepSeek V4-pro, and Qwen families) reveal a substantial gap between standalone and repository-level performance, as well as trade-offs between richer context and test reliability. Overall, XRepoTest provides a challenging and informative benchmark to advance scalable, robust, and multilingual unit test generation in realistic software environments.</abstract>
<identifier type="citekey">quang-etal-2026-xrepotest</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.274/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>6224</start>
<end>6262</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T XRepoTest: Benchmarking Multilingual Repository-Level Unit Test Generation for Large Language Models
%A Quang, Dung Le
%A Van, Dong Cao
%A Hai, Nam Le
%A Van, Linh Ngo
%A Bui, Thi-Mai-Anh
%A Nguyen, Phuong T.
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F quang-etal-2026-xrepotest
%X Large language models (LLMs) have shown promise for automated unit test generation, but existing evaluations largely rely on standalone settings and a narrow set of programming languages, overestimating real-world readiness. We introduce XRepoTest, a multilingual, repository-level benchmark for unit test generation spanning five underexplored languages: Rust, Go, Julia, PHP, and Ruby. XRepoTest evaluates tests under realistic repository constraints using a containerized execution framework and multiple context augmentation strategies, including file-level, LSP-based, and retrieval-based context. Beyond standard metrics such as test pass rate and coverage, we propose Invocation Rate (IR) to assess whether generated tests meaningfully exercise the intended functionality. Experiments with 14 state-of-the-art LLMs (e.g., Claude 4.5, GPT-5.2, DeepSeek V4-pro, and Qwen families) reveal a substantial gap between standalone and repository-level performance, as well as trade-offs between richer context and test reliability. Overall, XRepoTest provides a challenging and informative benchmark to advance scalable, robust, and multilingual unit test generation in realistic software environments.
%U https://aclanthology.org/2026.emnlp-main.274/
%P 6224-6262
Markdown (Informal)
[XRepoTest: Benchmarking Multilingual Repository-Level Unit Test Generation for Large Language Models](https://aclanthology.org/2026.emnlp-main.274/) (Quang et al., EMNLP 2026)
ACL