@inproceedings{el-hamdani-etal-2026-musart,
title = "{MUSART}: A Scalable Multi-Domain Benchmark for Measuring Factual Recall in Large Language Models",
author = "El Hamdani, Rajaa and
Bonald, Thomas and
Malliaros, Fragkiskos",
editor = "Kalo, Jan-Christoph and
Nakashole, Ndapa and
Razniewski, Simon and
Suchanek, Fabian M. and
Vlachos, Andreas and
McCallum, Andrew",
booktitle = "Proceedings of the 11th Workshop on Automated Knowledge Base Construction ({AKBC} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.akbc-1.12/",
pages = "151--168",
ISBN = "979-8-89176-464-4",
abstract = "Reliable evaluation of factual recall in large language models (LLMs) requires ground truth that is broad, clean, and complete. Yet existing benchmarks often rely on noisy extraction, narrow domains, or incomplete knowledge bases that penalize models for generating correct facts absent from the reference. We introduce **MUSART**, a benchmark of 112 596 factual triples over 31 domains, automatically constructed by pairing Wikipedia Good Article categories with Wikidata and selecting domain-salient relations via Personalized PageRank. To overcome knowledge-base incompleteness, we contribute an automated, character-grounded augmentation pipeline that mines verbatim facts from source articles. Evaluating twenty-one open-weight LLMs against both reference regimes reveals that (i) completeness correction systematically raises precision and lowers recall across all models, especially on multi-valued relations; and (ii) factual recall varies widely across subject domains, demonstrating why single-domain evaluation confounds model capability with subject matter. All code, data, and evaluation scripts are publicly released at \url{https://github.com/Rajjaa/MUSART}."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="el-hamdani-etal-2026-musart">
<titleInfo>
<title>MUSART: A Scalable Multi-Domain Benchmark for Measuring Factual Recall in Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Rajaa</namePart>
<namePart type="family">El Hamdani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thomas</namePart>
<namePart type="family">Bonald</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fragkiskos</namePart>
<namePart type="family">Malliaros</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 11th Workshop on Automated Knowledge Base Construction (AKBC 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jan-Christoph</namePart>
<namePart type="family">Kalo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ndapa</namePart>
<namePart type="family">Nakashole</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Razniewski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fabian</namePart>
<namePart type="given">M</namePart>
<namePart type="family">Suchanek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andreas</namePart>
<namePart type="family">Vlachos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andrew</namePart>
<namePart type="family">McCallum</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-464-4</identifier>
</relatedItem>
<abstract>Reliable evaluation of factual recall in large language models (LLMs) requires ground truth that is broad, clean, and complete. Yet existing benchmarks often rely on noisy extraction, narrow domains, or incomplete knowledge bases that penalize models for generating correct facts absent from the reference. We introduce **MUSART**, a benchmark of 112 596 factual triples over 31 domains, automatically constructed by pairing Wikipedia Good Article categories with Wikidata and selecting domain-salient relations via Personalized PageRank. To overcome knowledge-base incompleteness, we contribute an automated, character-grounded augmentation pipeline that mines verbatim facts from source articles. Evaluating twenty-one open-weight LLMs against both reference regimes reveals that (i) completeness correction systematically raises precision and lowers recall across all models, especially on multi-valued relations; and (ii) factual recall varies widely across subject domains, demonstrating why single-domain evaluation confounds model capability with subject matter. All code, data, and evaluation scripts are publicly released at https://github.com/Rajjaa/MUSART.</abstract>
<identifier type="citekey">el-hamdani-etal-2026-musart</identifier>
<location>
<url>https://aclanthology.org/2026.akbc-1.12/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>151</start>
<end>168</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MUSART: A Scalable Multi-Domain Benchmark for Measuring Factual Recall in Large Language Models
%A El Hamdani, Rajaa
%A Bonald, Thomas
%A Malliaros, Fragkiskos
%Y Kalo, Jan-Christoph
%Y Nakashole, Ndapa
%Y Razniewski, Simon
%Y Suchanek, Fabian M.
%Y Vlachos, Andreas
%Y McCallum, Andrew
%S Proceedings of the 11th Workshop on Automated Knowledge Base Construction (AKBC 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-464-4
%F el-hamdani-etal-2026-musart
%X Reliable evaluation of factual recall in large language models (LLMs) requires ground truth that is broad, clean, and complete. Yet existing benchmarks often rely on noisy extraction, narrow domains, or incomplete knowledge bases that penalize models for generating correct facts absent from the reference. We introduce **MUSART**, a benchmark of 112 596 factual triples over 31 domains, automatically constructed by pairing Wikipedia Good Article categories with Wikidata and selecting domain-salient relations via Personalized PageRank. To overcome knowledge-base incompleteness, we contribute an automated, character-grounded augmentation pipeline that mines verbatim facts from source articles. Evaluating twenty-one open-weight LLMs against both reference regimes reveals that (i) completeness correction systematically raises precision and lowers recall across all models, especially on multi-valued relations; and (ii) factual recall varies widely across subject domains, demonstrating why single-domain evaluation confounds model capability with subject matter. All code, data, and evaluation scripts are publicly released at https://github.com/Rajjaa/MUSART.
%U https://aclanthology.org/2026.akbc-1.12/
%P 151-168
Markdown (Informal)
[MUSART: A Scalable Multi-Domain Benchmark for Measuring Factual Recall in Large Language Models](https://aclanthology.org/2026.akbc-1.12/) (El Hamdani et al., AKBC 2026)
ACL