@inproceedings{ljapunov-etal-2026-deriving,
title = "Deriving Necessary Validity Requirements for Evaluating the Consistency of {LLM} Responses",
author = "Ljapunov, Alexander and
Kiehne, Niklas and
Balke, Wolf-Tilo",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2692/",
pages = "61246--61266",
ISBN = "979-8-89176-459-0",
abstract = "The probabilistic nature of large language models (LLMs) has regularly invited inquiries into their consistency with regard to repeated sampling, input variations, and conceptually related prompts. However, unlike accuracy benchmarks, consistency evaluations cannot rely on standardized measurement tools. We conduct a comprehensive survey of current methodologies evaluating language model consistency across 85 peer-reviewed publications, organizing them into a novel three-level taxonomy: (1) sampling, (2) semantic, and (3) relational consistency. Throughout this process, we identify three fundamental threats to the construct validity of LLM consistency measurements: Valid evaluations must disentangle intrinsic consistency from correctness, probabilistic uncertainty, and artifacts of constrained generation. Our systematic analysis shows that none of the 85 publications accounts for all three factors jointly, allowing construct-irrelevant variance to obscure our understanding of consistency."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ljapunov-etal-2026-deriving">
<titleInfo>
<title>Deriving Necessary Validity Requirements for Evaluating the Consistency of LLM Responses</title>
</titleInfo>
<name type="personal">
<namePart type="given">Alexander</namePart>
<namePart type="family">Ljapunov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Niklas</namePart>
<namePart type="family">Kiehne</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wolf-Tilo</namePart>
<namePart type="family">Balke</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>The probabilistic nature of large language models (LLMs) has regularly invited inquiries into their consistency with regard to repeated sampling, input variations, and conceptually related prompts. However, unlike accuracy benchmarks, consistency evaluations cannot rely on standardized measurement tools. We conduct a comprehensive survey of current methodologies evaluating language model consistency across 85 peer-reviewed publications, organizing them into a novel three-level taxonomy: (1) sampling, (2) semantic, and (3) relational consistency. Throughout this process, we identify three fundamental threats to the construct validity of LLM consistency measurements: Valid evaluations must disentangle intrinsic consistency from correctness, probabilistic uncertainty, and artifacts of constrained generation. Our systematic analysis shows that none of the 85 publications accounts for all three factors jointly, allowing construct-irrelevant variance to obscure our understanding of consistency.</abstract>
<identifier type="citekey">ljapunov-etal-2026-deriving</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2692/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>61246</start>
<end>61266</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Deriving Necessary Validity Requirements for Evaluating the Consistency of LLM Responses
%A Ljapunov, Alexander
%A Kiehne, Niklas
%A Balke, Wolf-Tilo
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F ljapunov-etal-2026-deriving
%X The probabilistic nature of large language models (LLMs) has regularly invited inquiries into their consistency with regard to repeated sampling, input variations, and conceptually related prompts. However, unlike accuracy benchmarks, consistency evaluations cannot rely on standardized measurement tools. We conduct a comprehensive survey of current methodologies evaluating language model consistency across 85 peer-reviewed publications, organizing them into a novel three-level taxonomy: (1) sampling, (2) semantic, and (3) relational consistency. Throughout this process, we identify three fundamental threats to the construct validity of LLM consistency measurements: Valid evaluations must disentangle intrinsic consistency from correctness, probabilistic uncertainty, and artifacts of constrained generation. Our systematic analysis shows that none of the 85 publications accounts for all three factors jointly, allowing construct-irrelevant variance to obscure our understanding of consistency.
%U https://aclanthology.org/2026.emnlp-main.2692/
%P 61246-61266
Markdown (Informal)
[Deriving Necessary Validity Requirements for Evaluating the Consistency of LLM Responses](https://aclanthology.org/2026.emnlp-main.2692/) (Ljapunov et al., EMNLP 2026)
ACL