@inproceedings{ping-etal-2026-llms,
title = "Why {LLM}s Give In: Conversational Factors and Reasoning Behind Medical Sycophancy",
author = "Ping, Kaike and
{\c{C}}ar{\i}k, Buse and
Wohn, Caleb and
Ding, Xiaohan and
Wang, Tongshuai and
Rho, Eugenia",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1558/",
doi = "10.18653/2026.findings-emnlp.1558",
pages = "33180--33201",
ISBN = "979-8-89176-462-0",
abstract = "Large language models can answer a medical question correctly and still abandon that answer when a user pushes back. We study this failure as medical sycophancy and ask when models are most likely to give in. Across five open-weight models, 500 MEDQUAD questions, and 1.2 million trials, we use a fully crossed design over four conversational factors: user role, user evidence, interaction structure, and grounding. Medical sycophancy is nearly three times more common when users challenge an answer the model has already given than when the false claim appears in the initial query. Models are also more susceptible to users presented as physicians or medical students. Most strikingly, fabricated evidence has opposite effects across interaction structures. It increases sycophancy in single-turn interactions but reduces it after the model has already answered. Grounding helps, but does not eliminate the behavior. Sycophancy varies more across medical questions than across models, making question selection an important part of benchmark design. Reasoning traces suggest that multi-turn failures are associated with models turning back toward their own prior answer, while fabricated evidence receives more scrutiny after an initial response. Together, the results show that medical sycophancy depends as much on how a model is challenged and evaluated as on which model is tested."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ping-etal-2026-llms">
<titleInfo>
<title>Why LLMs Give In: Conversational Factors and Reasoning Behind Medical Sycophancy</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kaike</namePart>
<namePart type="family">Ping</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Buse</namePart>
<namePart type="family">Çarık</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Caleb</namePart>
<namePart type="family">Wohn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaohan</namePart>
<namePart type="family">Ding</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tongshuai</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Eugenia</namePart>
<namePart type="family">Rho</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large language models can answer a medical question correctly and still abandon that answer when a user pushes back. We study this failure as medical sycophancy and ask when models are most likely to give in. Across five open-weight models, 500 MEDQUAD questions, and 1.2 million trials, we use a fully crossed design over four conversational factors: user role, user evidence, interaction structure, and grounding. Medical sycophancy is nearly three times more common when users challenge an answer the model has already given than when the false claim appears in the initial query. Models are also more susceptible to users presented as physicians or medical students. Most strikingly, fabricated evidence has opposite effects across interaction structures. It increases sycophancy in single-turn interactions but reduces it after the model has already answered. Grounding helps, but does not eliminate the behavior. Sycophancy varies more across medical questions than across models, making question selection an important part of benchmark design. Reasoning traces suggest that multi-turn failures are associated with models turning back toward their own prior answer, while fabricated evidence receives more scrutiny after an initial response. Together, the results show that medical sycophancy depends as much on how a model is challenged and evaluated as on which model is tested.</abstract>
<identifier type="citekey">ping-etal-2026-llms</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1558</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1558/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>33180</start>
<end>33201</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Why LLMs Give In: Conversational Factors and Reasoning Behind Medical Sycophancy
%A Ping, Kaike
%A Çarık, Buse
%A Wohn, Caleb
%A Ding, Xiaohan
%A Wang, Tongshuai
%A Rho, Eugenia
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F ping-etal-2026-llms
%X Large language models can answer a medical question correctly and still abandon that answer when a user pushes back. We study this failure as medical sycophancy and ask when models are most likely to give in. Across five open-weight models, 500 MEDQUAD questions, and 1.2 million trials, we use a fully crossed design over four conversational factors: user role, user evidence, interaction structure, and grounding. Medical sycophancy is nearly three times more common when users challenge an answer the model has already given than when the false claim appears in the initial query. Models are also more susceptible to users presented as physicians or medical students. Most strikingly, fabricated evidence has opposite effects across interaction structures. It increases sycophancy in single-turn interactions but reduces it after the model has already answered. Grounding helps, but does not eliminate the behavior. Sycophancy varies more across medical questions than across models, making question selection an important part of benchmark design. Reasoning traces suggest that multi-turn failures are associated with models turning back toward their own prior answer, while fabricated evidence receives more scrutiny after an initial response. Together, the results show that medical sycophancy depends as much on how a model is challenged and evaluated as on which model is tested.
%R 10.18653/2026.findings-emnlp.1558
%U https://aclanthology.org/2026.findings-emnlp.1558/
%U https://doi.org/10.18653/2026.findings-emnlp.1558
%P 33180-33201
Markdown (Informal)
[Why LLMs Give In: Conversational Factors and Reasoning Behind Medical Sycophancy](https://aclanthology.org/2026.findings-emnlp.1558/) (Ping et al., Findings 2026)
ACL