@inproceedings{kandlinger-etal-2026-llms,
title = "Do {LLM}s Give Consistent Opinions? Evaluating Response Reliability Under Varying {L}ikert-Scale Formulations in Survey-Style {MCQA}",
author = "Kandlinger, Martin and
Ma, Bolei and
Haensch, Anna-Carolina and
A{\ss}enmacher, Matthias",
editor = "Bak, JinYeong and
van der Goot, Rob and
Jang, Hyeju and
Buaphet, Weerayut and
Ramponi, Alan and
Xu, Wei and
Ritter, Alan",
booktitle = "Proceedings of the 11th Workshop on Natural User-generated Text",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.wnut-1.6/",
pages = "72--88",
ISBN = "979-8-89176-470-5",
abstract = "Survey methodology requires that equivalent question formulations elicit stable responses from the same respondent, regardless of how the Likert-type answer scale is presented. As LLMs are increasingly deployed as synthetic survey respondents, a critical but underexplored question arises: do they satisfy this basic parallel-forms reliability standard when answer scales vary in wording, cardinality, polarity, or option order? We extend the OpinionQA into two stress-test benchmarks, 1,235 subjective questions each paired with five reworded scale variants and six positional permutations, evaluated across four prompt-label formats. Using a four-metric consistency framework, we benchmark eight open-weight models from four families in both base and instruction-tuned configurations. Three findings emerge. First, base models fail basic reliability requirements: they exhibit severe mechanical instability driven by primacy bias, label sensitivity, and formatting failures, including full polarity inversions for the same question. Second, instruction tuning acts as a reliability filter, reducing response variance by approximately 50{\%} and enabling models to treat semantically equivalent scales as equivalent. Third, for instruction-tuned models under randomized option ordering, removing explicit labels paradoxically increases consistency by eliminating symbol-binding conflicts. These findings establish that response reliability is an engineered property of LLMs requiring both alignment training and deliberate prompt design, with concrete implications for survey-like MCQA evaluations."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kandlinger-etal-2026-llms">
<titleInfo>
<title>Do LLMs Give Consistent Opinions? Evaluating Response Reliability Under Varying Likert-Scale Formulations in Survey-Style MCQA</title>
</titleInfo>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Kandlinger</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bolei</namePart>
<namePart type="family">Ma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anna-Carolina</namePart>
<namePart type="family">Haensch</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Matthias</namePart>
<namePart type="family">Aßenmacher</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 11th Workshop on Natural User-generated Text</title>
</titleInfo>
<name type="personal">
<namePart type="given">JinYeong</namePart>
<namePart type="family">Bak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rob</namePart>
<namePart type="family">van der Goot</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hyeju</namePart>
<namePart type="family">Jang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weerayut</namePart>
<namePart type="family">Buaphet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Ramponi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alan</namePart>
<namePart type="family">Ritter</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-470-5</identifier>
</relatedItem>
<abstract>Survey methodology requires that equivalent question formulations elicit stable responses from the same respondent, regardless of how the Likert-type answer scale is presented. As LLMs are increasingly deployed as synthetic survey respondents, a critical but underexplored question arises: do they satisfy this basic parallel-forms reliability standard when answer scales vary in wording, cardinality, polarity, or option order? We extend the OpinionQA into two stress-test benchmarks, 1,235 subjective questions each paired with five reworded scale variants and six positional permutations, evaluated across four prompt-label formats. Using a four-metric consistency framework, we benchmark eight open-weight models from four families in both base and instruction-tuned configurations. Three findings emerge. First, base models fail basic reliability requirements: they exhibit severe mechanical instability driven by primacy bias, label sensitivity, and formatting failures, including full polarity inversions for the same question. Second, instruction tuning acts as a reliability filter, reducing response variance by approximately 50% and enabling models to treat semantically equivalent scales as equivalent. Third, for instruction-tuned models under randomized option ordering, removing explicit labels paradoxically increases consistency by eliminating symbol-binding conflicts. These findings establish that response reliability is an engineered property of LLMs requiring both alignment training and deliberate prompt design, with concrete implications for survey-like MCQA evaluations.</abstract>
<identifier type="citekey">kandlinger-etal-2026-llms</identifier>
<location>
<url>https://aclanthology.org/2026.wnut-1.6/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>72</start>
<end>88</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Do LLMs Give Consistent Opinions? Evaluating Response Reliability Under Varying Likert-Scale Formulations in Survey-Style MCQA
%A Kandlinger, Martin
%A Ma, Bolei
%A Haensch, Anna-Carolina
%A Aßenmacher, Matthias
%Y Bak, JinYeong
%Y van der Goot, Rob
%Y Jang, Hyeju
%Y Buaphet, Weerayut
%Y Ramponi, Alan
%Y Xu, Wei
%Y Ritter, Alan
%S Proceedings of the 11th Workshop on Natural User-generated Text
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-470-5
%F kandlinger-etal-2026-llms
%X Survey methodology requires that equivalent question formulations elicit stable responses from the same respondent, regardless of how the Likert-type answer scale is presented. As LLMs are increasingly deployed as synthetic survey respondents, a critical but underexplored question arises: do they satisfy this basic parallel-forms reliability standard when answer scales vary in wording, cardinality, polarity, or option order? We extend the OpinionQA into two stress-test benchmarks, 1,235 subjective questions each paired with five reworded scale variants and six positional permutations, evaluated across four prompt-label formats. Using a four-metric consistency framework, we benchmark eight open-weight models from four families in both base and instruction-tuned configurations. Three findings emerge. First, base models fail basic reliability requirements: they exhibit severe mechanical instability driven by primacy bias, label sensitivity, and formatting failures, including full polarity inversions for the same question. Second, instruction tuning acts as a reliability filter, reducing response variance by approximately 50% and enabling models to treat semantically equivalent scales as equivalent. Third, for instruction-tuned models under randomized option ordering, removing explicit labels paradoxically increases consistency by eliminating symbol-binding conflicts. These findings establish that response reliability is an engineered property of LLMs requiring both alignment training and deliberate prompt design, with concrete implications for survey-like MCQA evaluations.
%U https://aclanthology.org/2026.wnut-1.6/
%P 72-88
Markdown (Informal)
[Do LLMs Give Consistent Opinions? Evaluating Response Reliability Under Varying Likert-Scale Formulations in Survey-Style MCQA](https://aclanthology.org/2026.wnut-1.6/) (Kandlinger et al., WNUT 2026)
ACL