@inproceedings{morabito-etal-2026-rating,
title = "Rating the Pitch, Not the Product: User Evaluations of {LLM}s Reflect Expectations More Than Performance",
author = "Morabito, Robert and
McDonald, Tyler and
Viswanath, Charitra and
Hwang, Angel Hsing-Chi and
Gaube, Susanne and
Kabbara, Jad and
Emami, Ali",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.394/",
pages = "9039--9068",
ISBN = "979-8-89176-459-0",
abstract = "Imagine two users interact with the same LLM. One is told it is the cutting-edge flagship; the other, an older, weaker model. They walk away with markedly different ratings of its usefulness and intelligence, yet they used the *same model*. In a controlled study, 162 participants each used one of six LLMs from two families across three collaborative tasks, after first viewing a landing page that matched, overstated, or understated their model{'}s true capability. This pre-interaction *framing* persisted through use. Oversold users rated the model more favorably than Undersold users before interaction; afterward, both groups had updated toward reality, but the impression gap did not fully close. Framing also changed how users worked with the model (with Oversold users prompting more directively, Undersold users more collaboratively), but the quality of what they produced together depended only on the model{'}s true capability, not on what users were told. Participants' change in model impressions after use, measured across two impression measures, was *not* predicted by task performance ($\beta = -0.01$ and 0.11, both n.s.), but by whether the model met users' expectations ($\beta = 0.47$ and 0.50, both $p < .001$) and how confident they felt working with it ($\beta = 0.47$ and 0.36, both $p < .001$). After interaction, users are still rating the *pitch*, not the *product*: user-elicited LLM evaluations, including the preference data driving public leaderboards, measure expectation management at least as much as the model itself."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="morabito-etal-2026-rating">
<titleInfo>
<title>Rating the Pitch, Not the Product: User Evaluations of LLMs Reflect Expectations More Than Performance</title>
</titleInfo>
<name type="personal">
<namePart type="given">Robert</namePart>
<namePart type="family">Morabito</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tyler</namePart>
<namePart type="family">McDonald</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Charitra</namePart>
<namePart type="family">Viswanath</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Angel</namePart>
<namePart type="given">Hsing-Chi</namePart>
<namePart type="family">Hwang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Susanne</namePart>
<namePart type="family">Gaube</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jad</namePart>
<namePart type="family">Kabbara</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ali</namePart>
<namePart type="family">Emami</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Imagine two users interact with the same LLM. One is told it is the cutting-edge flagship; the other, an older, weaker model. They walk away with markedly different ratings of its usefulness and intelligence, yet they used the *same model*. In a controlled study, 162 participants each used one of six LLMs from two families across three collaborative tasks, after first viewing a landing page that matched, overstated, or understated their model’s true capability. This pre-interaction *framing* persisted through use. Oversold users rated the model more favorably than Undersold users before interaction; afterward, both groups had updated toward reality, but the impression gap did not fully close. Framing also changed how users worked with the model (with Oversold users prompting more directively, Undersold users more collaboratively), but the quality of what they produced together depended only on the model’s true capability, not on what users were told. Participants’ change in model impressions after use, measured across two impression measures, was *not* predicted by task performance (β = -0.01 and 0.11, both n.s.), but by whether the model met users’ expectations (β = 0.47 and 0.50, both p < .001) and how confident they felt working with it (β = 0.47 and 0.36, both p < .001). After interaction, users are still rating the *pitch*, not the *product*: user-elicited LLM evaluations, including the preference data driving public leaderboards, measure expectation management at least as much as the model itself.</abstract>
<identifier type="citekey">morabito-etal-2026-rating</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.394/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>9039</start>
<end>9068</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Rating the Pitch, Not the Product: User Evaluations of LLMs Reflect Expectations More Than Performance
%A Morabito, Robert
%A McDonald, Tyler
%A Viswanath, Charitra
%A Hwang, Angel Hsing-Chi
%A Gaube, Susanne
%A Kabbara, Jad
%A Emami, Ali
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F morabito-etal-2026-rating
%X Imagine two users interact with the same LLM. One is told it is the cutting-edge flagship; the other, an older, weaker model. They walk away with markedly different ratings of its usefulness and intelligence, yet they used the *same model*. In a controlled study, 162 participants each used one of six LLMs from two families across three collaborative tasks, after first viewing a landing page that matched, overstated, or understated their model’s true capability. This pre-interaction *framing* persisted through use. Oversold users rated the model more favorably than Undersold users before interaction; afterward, both groups had updated toward reality, but the impression gap did not fully close. Framing also changed how users worked with the model (with Oversold users prompting more directively, Undersold users more collaboratively), but the quality of what they produced together depended only on the model’s true capability, not on what users were told. Participants’ change in model impressions after use, measured across two impression measures, was *not* predicted by task performance (β = -0.01 and 0.11, both n.s.), but by whether the model met users’ expectations (β = 0.47 and 0.50, both p < .001) and how confident they felt working with it (β = 0.47 and 0.36, both p < .001). After interaction, users are still rating the *pitch*, not the *product*: user-elicited LLM evaluations, including the preference data driving public leaderboards, measure expectation management at least as much as the model itself.
%U https://aclanthology.org/2026.emnlp-main.394/
%P 9039-9068
Markdown (Informal)
[Rating the Pitch, Not the Product: User Evaluations of LLMs Reflect Expectations More Than Performance](https://aclanthology.org/2026.emnlp-main.394/) (Morabito et al., EMNLP 2026)
ACL
- Robert Morabito, Tyler McDonald, Charitra Viswanath, Angel Hsing-Chi Hwang, Susanne Gaube, Jad Kabbara, and Ali Emami. 2026. Rating the Pitch, Not the Product: User Evaluations of LLMs Reflect Expectations More Than Performance. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 9039–9068, Budapest, Hungary. Association for Computational Linguistics.