@inproceedings{karakas-simsek-2026-evidence,
title = "Not All Evidence Is Equal: Source-Sensitive Calibration in Large Language Models",
author = "Karaka{\c{s}}, Sercan and
{\c{S}}im{\c{s}}ek, Yusuf",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1306/",
pages = "29730--29749",
ISBN = "979-8-89176-459-0",
abstract = "Large language models are increasingly used in settings where information must be evaluated not only for content, but also for reliability. This paper introduces a human-grounded evaluation[All data and code used in this paper will be made available with the camera-ready version.] of source-sensitive calibration in Turkish and English. Across hearer-side uptake tasks, we manipulate source reliability while holding propositional content constant, and test whether humans and LLMs adjust action-readiness judgments according to source trustworthiness. Turkish human participants show robust source sensitivity: they are substantially more willing to act on information attributed to high-trust sources than to low-trust sources, while Turkish evidential morphology itself does not independently affect uptake once the source is overtly specified. LLMs show weaker calibration. In Turkish, models partially register explicit source reliability, but the effect is much smaller than in humans and there is no reliable effect of the \textit{-DI}/\textit{-mI{\c{s}}} contrast. In English, models show small effects of both source reliability and the lexical marker \textit{apparently}, but no robust interaction between them. These results suggest that current LLMs encode source cues only partially and often fail to convert them into human-like judgments about when information is reliable enough to guide action."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="karakas-simsek-2026-evidence">
<titleInfo>
<title>Not All Evidence Is Equal: Source-Sensitive Calibration in Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sercan</namePart>
<namePart type="family">Karakaş</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yusuf</namePart>
<namePart type="family">Şimşek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large language models are increasingly used in settings where information must be evaluated not only for content, but also for reliability. This paper introduces a human-grounded evaluation[All data and code used in this paper will be made available with the camera-ready version.] of source-sensitive calibration in Turkish and English. Across hearer-side uptake tasks, we manipulate source reliability while holding propositional content constant, and test whether humans and LLMs adjust action-readiness judgments according to source trustworthiness. Turkish human participants show robust source sensitivity: they are substantially more willing to act on information attributed to high-trust sources than to low-trust sources, while Turkish evidential morphology itself does not independently affect uptake once the source is overtly specified. LLMs show weaker calibration. In Turkish, models partially register explicit source reliability, but the effect is much smaller than in humans and there is no reliable effect of the -DI/-mIş contrast. In English, models show small effects of both source reliability and the lexical marker apparently, but no robust interaction between them. These results suggest that current LLMs encode source cues only partially and often fail to convert them into human-like judgments about when information is reliable enough to guide action.</abstract>
<identifier type="citekey">karakas-simsek-2026-evidence</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1306/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>29730</start>
<end>29749</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Not All Evidence Is Equal: Source-Sensitive Calibration in Large Language Models
%A Karakaş, Sercan
%A Şimşek, Yusuf
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F karakas-simsek-2026-evidence
%X Large language models are increasingly used in settings where information must be evaluated not only for content, but also for reliability. This paper introduces a human-grounded evaluation[All data and code used in this paper will be made available with the camera-ready version.] of source-sensitive calibration in Turkish and English. Across hearer-side uptake tasks, we manipulate source reliability while holding propositional content constant, and test whether humans and LLMs adjust action-readiness judgments according to source trustworthiness. Turkish human participants show robust source sensitivity: they are substantially more willing to act on information attributed to high-trust sources than to low-trust sources, while Turkish evidential morphology itself does not independently affect uptake once the source is overtly specified. LLMs show weaker calibration. In Turkish, models partially register explicit source reliability, but the effect is much smaller than in humans and there is no reliable effect of the -DI/-mIş contrast. In English, models show small effects of both source reliability and the lexical marker apparently, but no robust interaction between them. These results suggest that current LLMs encode source cues only partially and often fail to convert them into human-like judgments about when information is reliable enough to guide action.
%U https://aclanthology.org/2026.emnlp-main.1306/
%P 29730-29749
Markdown (Informal)
[Not All Evidence Is Equal: Source-Sensitive Calibration in Large Language Models](https://aclanthology.org/2026.emnlp-main.1306/) (Karakaş & Şimşek, EMNLP 2026)
ACL