@inproceedings{cecil-freedman-2026-beyond,
title = "Beyond Majority Vote: Multi-Perspective Adjudication for Medical Hallucination Detection",
author = "Cecil, Joe and
Freedman, Marjorie",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1935/",
doi = "10.18653/2026.findings-emnlp.1935",
pages = "41258--41291",
ISBN = "979-8-89176-462-0",
abstract = "Understanding the frequency of factual errors in chatbot-generated text and evaluating systems that detect these errors is critical for determining chatbot safety. Yet factual-error detection is often treated as a single-pass, single-annotator labeling problem. In long-form chatbot responses, factual errors can be subtle and embedded within mostly correct text.\par We develop a multi-perspective annotation study of medically relevant chatbot responses, combining first-pass annotation, LLM-as-a-Judge candidate discovery, and two forms of adjudication: domain-expert and evidence-based fact-checking. First-pass annotators frequently miss factual errors later validated by adjudicators. LLM-as-a-Judge improves candidate discovery, but is insufficient on its own: it misses factual errors that annotators catch. We also find disagreement among adjudicators, suggesting that adjudication over multiple candidate sources can improve benchmark completeness, but does not eliminate the need to apply judgment and expertise. Applied to an existing benchmark, this technique reveals a similar pattern of missed errors.\par Together, these results suggest that single-pass hallucination benchmarks achieve scale, but likely undercount errors. Multi-pass adjudication improves coverage, but benchmark results will still need to be interpreted in light of the judgment, expertise, and evidence used when deciding if an error is present."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="cecil-freedman-2026-beyond">
<titleInfo>
<title>Beyond Majority Vote: Multi-Perspective Adjudication for Medical Hallucination Detection</title>
</titleInfo>
<name type="personal">
<namePart type="given">Joe</namePart>
<namePart type="family">Cecil</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marjorie</namePart>
<namePart type="family">Freedman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Understanding the frequency of factual errors in chatbot-generated text and evaluating systems that detect these errors is critical for determining chatbot safety. Yet factual-error detection is often treated as a single-pass, single-annotator labeling problem. In long-form chatbot responses, factual errors can be subtle and embedded within mostly correct text.\par We develop a multi-perspective annotation study of medically relevant chatbot responses, combining first-pass annotation, LLM-as-a-Judge candidate discovery, and two forms of adjudication: domain-expert and evidence-based fact-checking. First-pass annotators frequently miss factual errors later validated by adjudicators. LLM-as-a-Judge improves candidate discovery, but is insufficient on its own: it misses factual errors that annotators catch. We also find disagreement among adjudicators, suggesting that adjudication over multiple candidate sources can improve benchmark completeness, but does not eliminate the need to apply judgment and expertise. Applied to an existing benchmark, this technique reveals a similar pattern of missed errors.\par Together, these results suggest that single-pass hallucination benchmarks achieve scale, but likely undercount errors. Multi-pass adjudication improves coverage, but benchmark results will still need to be interpreted in light of the judgment, expertise, and evidence used when deciding if an error is present.</abstract>
<identifier type="citekey">cecil-freedman-2026-beyond</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1935</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1935/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>41258</start>
<end>41291</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Majority Vote: Multi-Perspective Adjudication for Medical Hallucination Detection
%A Cecil, Joe
%A Freedman, Marjorie
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F cecil-freedman-2026-beyond
%X Understanding the frequency of factual errors in chatbot-generated text and evaluating systems that detect these errors is critical for determining chatbot safety. Yet factual-error detection is often treated as a single-pass, single-annotator labeling problem. In long-form chatbot responses, factual errors can be subtle and embedded within mostly correct text.\par We develop a multi-perspective annotation study of medically relevant chatbot responses, combining first-pass annotation, LLM-as-a-Judge candidate discovery, and two forms of adjudication: domain-expert and evidence-based fact-checking. First-pass annotators frequently miss factual errors later validated by adjudicators. LLM-as-a-Judge improves candidate discovery, but is insufficient on its own: it misses factual errors that annotators catch. We also find disagreement among adjudicators, suggesting that adjudication over multiple candidate sources can improve benchmark completeness, but does not eliminate the need to apply judgment and expertise. Applied to an existing benchmark, this technique reveals a similar pattern of missed errors.\par Together, these results suggest that single-pass hallucination benchmarks achieve scale, but likely undercount errors. Multi-pass adjudication improves coverage, but benchmark results will still need to be interpreted in light of the judgment, expertise, and evidence used when deciding if an error is present.
%R 10.18653/2026.findings-emnlp.1935
%U https://aclanthology.org/2026.findings-emnlp.1935/
%U https://doi.org/10.18653/2026.findings-emnlp.1935
%P 41258-41291
Markdown (Informal)
[Beyond Majority Vote: Multi-Perspective Adjudication for Medical Hallucination Detection](https://aclanthology.org/2026.findings-emnlp.1935/) (Cecil & Freedman, Findings 2026)
ACL