@inproceedings{contreras-etal-2026-plausible,
title = "Plausible but Wrong: A Statistical Evaluation of {GPT} Models on High-Stakes Migration Statistics",
author = "Contreras, Hector and
Annapureddy, Ravinithesh and
Gatica-perez, Daniel",
editor = "Xu, Shanshan and
T.Y.S.S., Santosh and
Kurniawan, Kemal and
Peng, Siyao and
Wallbridge, Sarenne and
Koto, Fajri and
Dziri, Nouha and
Plank, Barbara",
booktitle = "Proceedings of the First Workshop on Pluralistic {AI} {NLP}: Diversity-aware, Sociotechnical, Responsible Alignment ({PANDORA} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.pandora-1.22/",
pages = "388--403",
ISBN = "979-8-89176-484-2",
abstract = "LLMs increasingly serve as general-purpose information systems, including for factual queries about public statistics. In high-stakes domains such as migration, where data accuracy shapes both policy and public attitudes toward at-risk populations, models may answer queries about social groups unequally. We evaluate this concern empirically using GPT-4 Turbo, GPT-4o, GPT-5 chat, and GPT-5.1 on migration-related education statistics. Each model receives zero-shot queries on educational-attainment distributions for national and foreign citizens aged 25{--}54 across 34 European countries from 2020{--}2023. We compare outputs against official Eurostat data using RMSE, total variation distance (TVD), cosine similarity, distributional analysis, and response-coverage rates. Across models, results show a consistent gap between plausibility and accuracy: all models preserve broad distributional structure (cosine similarity {\ensuremath{>}} 0.93), while deviating substantially from ground-truth values (RMSE 10{--}14 and TVD 12{--}16 percentage points). We characterize this combination of structural plausibility and numerical error as an output-level analog of the illusion of competence. GPT-4 models also produce explicit non-answers for 30{--}78{\%} of responses, with refusals (explicit non-answers) up to 3.4 times more frequent for foreigners than for nationals. GPT-5 models eliminate refusals but return hallucinated values. These findings show that pluralistic evaluation must distinguish legitimate human variation from unsupported model variation and assess whether factual information remains comparably available and reliable across social groups."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="contreras-etal-2026-plausible">
<titleInfo>
<title>Plausible but Wrong: A Statistical Evaluation of GPT Models on High-Stakes Migration Statistics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hector</namePart>
<namePart type="family">Contreras</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ravinithesh</namePart>
<namePart type="family">Annapureddy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Gatica-perez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shanshan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Santosh</namePart>
<namePart type="family">T.Y.S.S.</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kemal</namePart>
<namePart type="family">Kurniawan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siyao</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sarenne</namePart>
<namePart type="family">Wallbridge</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fajri</namePart>
<namePart type="family">Koto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nouha</namePart>
<namePart type="family">Dziri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-484-2</identifier>
</relatedItem>
<abstract>LLMs increasingly serve as general-purpose information systems, including for factual queries about public statistics. In high-stakes domains such as migration, where data accuracy shapes both policy and public attitudes toward at-risk populations, models may answer queries about social groups unequally. We evaluate this concern empirically using GPT-4 Turbo, GPT-4o, GPT-5 chat, and GPT-5.1 on migration-related education statistics. Each model receives zero-shot queries on educational-attainment distributions for national and foreign citizens aged 25–54 across 34 European countries from 2020–2023. We compare outputs against official Eurostat data using RMSE, total variation distance (TVD), cosine similarity, distributional analysis, and response-coverage rates. Across models, results show a consistent gap between plausibility and accuracy: all models preserve broad distributional structure (cosine similarity \ensuremath> 0.93), while deviating substantially from ground-truth values (RMSE 10–14 and TVD 12–16 percentage points). We characterize this combination of structural plausibility and numerical error as an output-level analog of the illusion of competence. GPT-4 models also produce explicit non-answers for 30–78% of responses, with refusals (explicit non-answers) up to 3.4 times more frequent for foreigners than for nationals. GPT-5 models eliminate refusals but return hallucinated values. These findings show that pluralistic evaluation must distinguish legitimate human variation from unsupported model variation and assess whether factual information remains comparably available and reliable across social groups.</abstract>
<identifier type="citekey">contreras-etal-2026-plausible</identifier>
<location>
<url>https://aclanthology.org/2026.pandora-1.22/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>388</start>
<end>403</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Plausible but Wrong: A Statistical Evaluation of GPT Models on High-Stakes Migration Statistics
%A Contreras, Hector
%A Annapureddy, Ravinithesh
%A Gatica-perez, Daniel
%Y Xu, Shanshan
%Y T.Y.S.S., Santosh
%Y Kurniawan, Kemal
%Y Peng, Siyao
%Y Wallbridge, Sarenne
%Y Koto, Fajri
%Y Dziri, Nouha
%Y Plank, Barbara
%S Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-484-2
%F contreras-etal-2026-plausible
%X LLMs increasingly serve as general-purpose information systems, including for factual queries about public statistics. In high-stakes domains such as migration, where data accuracy shapes both policy and public attitudes toward at-risk populations, models may answer queries about social groups unequally. We evaluate this concern empirically using GPT-4 Turbo, GPT-4o, GPT-5 chat, and GPT-5.1 on migration-related education statistics. Each model receives zero-shot queries on educational-attainment distributions for national and foreign citizens aged 25–54 across 34 European countries from 2020–2023. We compare outputs against official Eurostat data using RMSE, total variation distance (TVD), cosine similarity, distributional analysis, and response-coverage rates. Across models, results show a consistent gap between plausibility and accuracy: all models preserve broad distributional structure (cosine similarity \ensuremath> 0.93), while deviating substantially from ground-truth values (RMSE 10–14 and TVD 12–16 percentage points). We characterize this combination of structural plausibility and numerical error as an output-level analog of the illusion of competence. GPT-4 models also produce explicit non-answers for 30–78% of responses, with refusals (explicit non-answers) up to 3.4 times more frequent for foreigners than for nationals. GPT-5 models eliminate refusals but return hallucinated values. These findings show that pluralistic evaluation must distinguish legitimate human variation from unsupported model variation and assess whether factual information remains comparably available and reliable across social groups.
%U https://aclanthology.org/2026.pandora-1.22/
%P 388-403
Markdown (Informal)
[Plausible but Wrong: A Statistical Evaluation of GPT Models on High-Stakes Migration Statistics](https://aclanthology.org/2026.pandora-1.22/) (Contreras et al., PANDORA 2026)
ACL