@inproceedings{kuramoto-etal-2026-counts,
title = "What Counts as Hallucination? A Scoping Review of Tasks, Symptoms, and Convergence Expectations in {LLM} Research",
author = "Kuramoto, Toshiki and
Kamei, Ryohei and
Suzuki, Jun",
editor = {Aziz, Wilker and
Berant, Jonathan and
Eikema, Bryan and
Ilia, Evgenia and
de Marneffe, Marie-Catherine and
Plank, Barbara and
Shelmanov, Artem and
Swayamdipta, Swabha and
Tiedemann, J{\"o}rg and
Vazhentsev, Artem and
V{\'a}zquez, Ra{\'u}l and
Zerva, Chrysoula},
booktitle = "The Proceedings of the Third Workshop on Uncertainty-Aware {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.uncertainlp-main.38/",
pages = "625--656",
ISBN = "979-8-89176-486-6",
abstract = "Hallucination in large language models (LLMs) and related generative models has become a central issue for reliability and practical deployment. Yet prior work does not consistently define what is treated as hallucination or the level of expectation under which model outputs are evaluated. This study conducts a scoping review of 244 papers published between 2018 and 2025, analyzing definitions of hallucination, output-side symptoms, and Convergence Expectation (CE). The results show that approximately 90{\%} of the papers provide some form of operational definition, but the symptoms targeted by these operationalizations vary substantially by task. ``Addition'' appears across many tasks as a common symptom, whereas ``Transformation'' and ``Deviation'' strongly depend on task structure and input constraints. Analyses by CE band further show that, even within the same task, the symptoms problematized as hallucination vary according to the strictness of expected output convergence. These findings indicate that hallucination is not a fixed failure inherent in model outputs, but a phenomenon shaped by task, input constraints, evaluation criteria, and the expectations of humans or evaluators. This study reframes hallucination as expectation mismatch and argues that definitions, symptom scope, and expectation levels must be made explicit when comparing research findings and designing evaluations."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kuramoto-etal-2026-counts">
<titleInfo>
<title>What Counts as Hallucination? A Scoping Review of Tasks, Symptoms, and Convergence Expectations in LLM Research</title>
</titleInfo>
<name type="personal">
<namePart type="given">Toshiki</namePart>
<namePart type="family">Kuramoto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryohei</namePart>
<namePart type="family">Kamei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>The Proceedings of the Third Workshop on Uncertainty-Aware NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Wilker</namePart>
<namePart type="family">Aziz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jonathan</namePart>
<namePart type="family">Berant</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bryan</namePart>
<namePart type="family">Eikema</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Evgenia</namePart>
<namePart type="family">Ilia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marie-Catherine</namePart>
<namePart type="family">de Marneffe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Artem</namePart>
<namePart type="family">Shelmanov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Swabha</namePart>
<namePart type="family">Swayamdipta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jörg</namePart>
<namePart type="family">Tiedemann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Artem</namePart>
<namePart type="family">Vazhentsev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raúl</namePart>
<namePart type="family">Vázquez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chrysoula</namePart>
<namePart type="family">Zerva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-486-6</identifier>
</relatedItem>
<abstract>Hallucination in large language models (LLMs) and related generative models has become a central issue for reliability and practical deployment. Yet prior work does not consistently define what is treated as hallucination or the level of expectation under which model outputs are evaluated. This study conducts a scoping review of 244 papers published between 2018 and 2025, analyzing definitions of hallucination, output-side symptoms, and Convergence Expectation (CE). The results show that approximately 90% of the papers provide some form of operational definition, but the symptoms targeted by these operationalizations vary substantially by task. “Addition” appears across many tasks as a common symptom, whereas “Transformation” and “Deviation” strongly depend on task structure and input constraints. Analyses by CE band further show that, even within the same task, the symptoms problematized as hallucination vary according to the strictness of expected output convergence. These findings indicate that hallucination is not a fixed failure inherent in model outputs, but a phenomenon shaped by task, input constraints, evaluation criteria, and the expectations of humans or evaluators. This study reframes hallucination as expectation mismatch and argues that definitions, symptom scope, and expectation levels must be made explicit when comparing research findings and designing evaluations.</abstract>
<identifier type="citekey">kuramoto-etal-2026-counts</identifier>
<location>
<url>https://aclanthology.org/2026.uncertainlp-main.38/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>625</start>
<end>656</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What Counts as Hallucination? A Scoping Review of Tasks, Symptoms, and Convergence Expectations in LLM Research
%A Kuramoto, Toshiki
%A Kamei, Ryohei
%A Suzuki, Jun
%Y Aziz, Wilker
%Y Berant, Jonathan
%Y Eikema, Bryan
%Y Ilia, Evgenia
%Y de Marneffe, Marie-Catherine
%Y Plank, Barbara
%Y Shelmanov, Artem
%Y Swayamdipta, Swabha
%Y Tiedemann, Jörg
%Y Vazhentsev, Artem
%Y Vázquez, Raúl
%Y Zerva, Chrysoula
%S The Proceedings of the Third Workshop on Uncertainty-Aware NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-486-6
%F kuramoto-etal-2026-counts
%X Hallucination in large language models (LLMs) and related generative models has become a central issue for reliability and practical deployment. Yet prior work does not consistently define what is treated as hallucination or the level of expectation under which model outputs are evaluated. This study conducts a scoping review of 244 papers published between 2018 and 2025, analyzing definitions of hallucination, output-side symptoms, and Convergence Expectation (CE). The results show that approximately 90% of the papers provide some form of operational definition, but the symptoms targeted by these operationalizations vary substantially by task. “Addition” appears across many tasks as a common symptom, whereas “Transformation” and “Deviation” strongly depend on task structure and input constraints. Analyses by CE band further show that, even within the same task, the symptoms problematized as hallucination vary according to the strictness of expected output convergence. These findings indicate that hallucination is not a fixed failure inherent in model outputs, but a phenomenon shaped by task, input constraints, evaluation criteria, and the expectations of humans or evaluators. This study reframes hallucination as expectation mismatch and argues that definitions, symptom scope, and expectation levels must be made explicit when comparing research findings and designing evaluations.
%U https://aclanthology.org/2026.uncertainlp-main.38/
%P 625-656
Markdown (Informal)
[What Counts as Hallucination? A Scoping Review of Tasks, Symptoms, and Convergence Expectations in LLM Research](https://aclanthology.org/2026.uncertainlp-main.38/) (Kuramoto et al., UncertaiNLP 2026)
ACL