@inproceedings{rastegar-etal-2026-taxonomy,
title = "A Taxonomy of Safety: Harmonizing {LLM} Benchmarks in a Fragmented Landscape",
author = "Rastegar, Shadi and
Hangya, Viktor and
Kuech, Fabian and
Gold, Darina",
editor = "Piperidis, Stelios and
Bel, N{\'u}ria and
van den Heuvel, Henk and
Ide, Nancy and
Krek, Simon and
Toral, Antonio",
booktitle = "Proceedings of the Fifteenth Language Resources and Evaluation Conference",
month = may,
year = "2026",
address = "Palma de Mallorca, Spain",
publisher = "ELRA Language Resource Association",
url = "https://aclanthology.org/2026.lrec-1.350/",
doi = "10.63317/4n7jrxunmvcp",
pages = "4470--4481",
abstract = "Understanding and mitigating the safety limitations of LLMs is of great importance to build trustworthy AI applications. Although a wide range of safety benchmarks are available, there is no standardized taxonomy of safety categories. As a result, some benchmarks focus on a specific subset of categories, they define test samples on different granularity levels, or they use different definitions or naming conventions. To mitigate these issues, we propose a two-level taxonomy of LLM safety categories, created by harmonizing existing resources. Our taxonomy gives an overview of important safety categories that helps researchers pinpoint potential safety risks and select the right benchmarks when evaluating or developing language models. Moreover, the taxonomy provides guidelines to categorize future benchmarks. Furthermore, since the majority of the available safety resources are English-focused, we check the cross-cultural validity of our taxonomy by translating datasets covering all top level categories to French, German, Italian, and Spanish. A manual review of a subset of translated samples by native speakers revealed no major cultural mismatches from a safety perspective. This supports not only the transferability of English benchmarks but also the transferability of the categories in our taxonomy, as well as its potential as a practical tool for guiding safety-focused dataset development and evaluation beyond English."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="rastegar-etal-2026-taxonomy">
<titleInfo>
<title>A Taxonomy of Safety: Harmonizing LLM Benchmarks in a Fragmented Landscape</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shadi</namePart>
<namePart type="family">Rastegar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Viktor</namePart>
<namePart type="family">Hangya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fabian</namePart>
<namePart type="family">Kuech</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Darina</namePart>
<namePart type="family">Gold</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Fifteenth Language Resources and Evaluation Conference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Stelios</namePart>
<namePart type="family">Piperidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Núria</namePart>
<namePart type="family">Bel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Henk</namePart>
<namePart type="family">van den Heuvel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nancy</namePart>
<namePart type="family">Ide</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Krek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Antonio</namePart>
<namePart type="family">Toral</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resource Association</publisher>
<place>
<placeTerm type="text">Palma de Mallorca, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Understanding and mitigating the safety limitations of LLMs is of great importance to build trustworthy AI applications. Although a wide range of safety benchmarks are available, there is no standardized taxonomy of safety categories. As a result, some benchmarks focus on a specific subset of categories, they define test samples on different granularity levels, or they use different definitions or naming conventions. To mitigate these issues, we propose a two-level taxonomy of LLM safety categories, created by harmonizing existing resources. Our taxonomy gives an overview of important safety categories that helps researchers pinpoint potential safety risks and select the right benchmarks when evaluating or developing language models. Moreover, the taxonomy provides guidelines to categorize future benchmarks. Furthermore, since the majority of the available safety resources are English-focused, we check the cross-cultural validity of our taxonomy by translating datasets covering all top level categories to French, German, Italian, and Spanish. A manual review of a subset of translated samples by native speakers revealed no major cultural mismatches from a safety perspective. This supports not only the transferability of English benchmarks but also the transferability of the categories in our taxonomy, as well as its potential as a practical tool for guiding safety-focused dataset development and evaluation beyond English.</abstract>
<identifier type="citekey">rastegar-etal-2026-taxonomy</identifier>
<identifier type="doi">10.63317/4n7jrxunmvcp</identifier>
<location>
<url>https://aclanthology.org/2026.lrec-1.350/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>4470</start>
<end>4481</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T A Taxonomy of Safety: Harmonizing LLM Benchmarks in a Fragmented Landscape
%A Rastegar, Shadi
%A Hangya, Viktor
%A Kuech, Fabian
%A Gold, Darina
%Y Piperidis, Stelios
%Y Bel, Núria
%Y van den Heuvel, Henk
%Y Ide, Nancy
%Y Krek, Simon
%Y Toral, Antonio
%S Proceedings of the Fifteenth Language Resources and Evaluation Conference
%D 2026
%8 May
%I ELRA Language Resource Association
%C Palma de Mallorca, Spain
%F rastegar-etal-2026-taxonomy
%X Understanding and mitigating the safety limitations of LLMs is of great importance to build trustworthy AI applications. Although a wide range of safety benchmarks are available, there is no standardized taxonomy of safety categories. As a result, some benchmarks focus on a specific subset of categories, they define test samples on different granularity levels, or they use different definitions or naming conventions. To mitigate these issues, we propose a two-level taxonomy of LLM safety categories, created by harmonizing existing resources. Our taxonomy gives an overview of important safety categories that helps researchers pinpoint potential safety risks and select the right benchmarks when evaluating or developing language models. Moreover, the taxonomy provides guidelines to categorize future benchmarks. Furthermore, since the majority of the available safety resources are English-focused, we check the cross-cultural validity of our taxonomy by translating datasets covering all top level categories to French, German, Italian, and Spanish. A manual review of a subset of translated samples by native speakers revealed no major cultural mismatches from a safety perspective. This supports not only the transferability of English benchmarks but also the transferability of the categories in our taxonomy, as well as its potential as a practical tool for guiding safety-focused dataset development and evaluation beyond English.
%R 10.63317/4n7jrxunmvcp
%U https://aclanthology.org/2026.lrec-1.350/
%U https://doi.org/10.63317/4n7jrxunmvcp
%P 4470-4481
Markdown (Informal)
[A Taxonomy of Safety: Harmonizing LLM Benchmarks in a Fragmented Landscape](https://aclanthology.org/2026.lrec-1.350/) (Rastegar et al., LREC 2026)
ACL